Home | History | Annotate | Download | only in base 
      1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 """Module containing base test results classes."""
      6 
      7 class ResultType(object):
      8   """Class enumerating test types."""
      9   PASS = 'PASS'
     10   SKIP = 'SKIP'
     11   FAIL = 'FAIL'
     12   CRASH = 'CRASH'
     13   TIMEOUT = 'TIMEOUT'
     14   UNKNOWN = 'UNKNOWN'
     15 
     16   @staticmethod
     17   def GetTypes():
     18     """Get a list of all test types."""
     19     return [ResultType.PASS, ResultType.SKIP, ResultType.FAIL,
     20             ResultType.CRASH, ResultType.TIMEOUT, ResultType.UNKNOWN]
     21 
     22 
     23 class BaseTestResult(object):
     24   """Base class for a single test result."""
     25 
     26   def __init__(self, name, test_type, log=''):
     27     """Construct a BaseTestResult.
     28 
     29     Args:
     30       name: Name of the test which defines uniqueness.
     31       test_type: Type of the test result as defined in ResultType.
     32       log: An optional string listing any errors.
     33     """
     34     assert name
     35     assert test_type in ResultType.GetTypes()
     36     self._name = name
     37     self._test_type = test_type
     38     self._log = log
     39 
     40   def __str__(self):
     41     return self._name
     42 
     43   def __repr__(self):
     44     return self._name
     45 
     46   def __cmp__(self, other):
     47     # pylint: disable=W0212
     48     return cmp(self._name, other._name)
     49 
     50   def __hash__(self):
     51     return hash(self._name)
     52 
     53   def SetName(self, name):
     54     """Set the test name.
     55 
     56     Because we're putting this into a set, this should only be used if moving
     57     this test result into another set.
     58     """
     59     self._name = name
     60 
     61   def GetName(self):
     62     """Get the test name."""
     63     return self._name
     64 
     65   def GetType(self):
     66     """Get the test result type."""
     67     return self._test_type
     68 
     69   def GetLog(self):
     70     """Get the test log."""
     71     return self._log
     72 
     73 
     74 class TestRunResults(object):
     75   """Set of results for a test run."""
     76 
     77   def __init__(self):
     78     self._results = set()
     79 
     80   def GetLogs(self):
     81     """Get the string representation of all test logs."""
     82     s = []
     83     for test_type in ResultType.GetTypes():
     84       if test_type != ResultType.PASS:
     85         for t in sorted(self._GetType(test_type)):
     86           log = t.GetLog()
     87           if log:
     88             s.append('[%s] %s:' % (test_type, t))
     89             s.append(log)
     90     return '\n'.join(s)
     91 
     92   def GetGtestForm(self):
     93     """Get the gtest string representation of this object."""
     94     s = []
     95     plural = lambda n, s, p: '%d %s' % (n, p if n != 1 else s)
     96     tests = lambda n: plural(n, 'test', 'tests')
     97 
     98     s.append('[==========] %s ran.' % (tests(len(self.GetAll()))))
     99     s.append('[  PASSED  ] %s.' % (tests(len(self.GetPass()))))
    100 
    101     skipped = self.GetSkip()
    102     if skipped:
    103       s.append('[  SKIPPED ] Skipped %s, listed below:' % tests(len(skipped)))
    104       for t in sorted(skipped):
    105         s.append('[  SKIPPED ] %s' % str(t))
    106 
    107     all_failures = self.GetFail().union(self.GetCrash(), self.GetTimeout(),
    108         self.GetUnknown())
    109     if all_failures:
    110       s.append('[  FAILED  ] %s, listed below:' % tests(len(all_failures)))
    111       for t in sorted(self.GetFail()):
    112         s.append('[  FAILED  ] %s' % str(t))
    113       for t in sorted(self.GetCrash()):
    114         s.append('[  FAILED  ] %s (CRASHED)' % str(t))
    115       for t in sorted(self.GetTimeout()):
    116         s.append('[  FAILED  ] %s (TIMEOUT)' % str(t))
    117       for t in sorted(self.GetUnknown()):
    118         s.append('[  FAILED  ] %s (UNKNOWN)' % str(t))
    119       s.append('')
    120       s.append(plural(len(all_failures), 'FAILED TEST', 'FAILED TESTS'))
    121     return '\n'.join(s)
    122 
    123   def GetShortForm(self):
    124     """Get the short string representation of this object."""
    125     s = []
    126     s.append('ALL: %d' % len(self._results))
    127     for test_type in ResultType.GetTypes():
    128       s.append('%s: %d' % (test_type, len(self._GetType(test_type))))
    129     return ''.join([x.ljust(15) for x in s])
    130 
    131   def __str__(self):
    132     return self.GetLongForm()
    133 
    134   def AddResult(self, result):
    135     """Add |result| to the set.
    136 
    137     Args:
    138       result: An instance of BaseTestResult.
    139     """
    140     assert isinstance(result, BaseTestResult)
    141     self._results.add(result)
    142 
    143   def AddResults(self, results):
    144     """Add |results| to the set.
    145 
    146     Args:
    147       results: An iterable of BaseTestResult objects.
    148     """
    149     for t in results:
    150       self.AddResult(t)
    151 
    152   def AddTestRunResults(self, results):
    153     """Add the set of test results from |results|.
    154 
    155     Args:
    156       results: An instance of TestRunResults.
    157     """
    158     assert isinstance(results, TestRunResults)
    159     # pylint: disable=W0212
    160     self._results.update(results._results)
    161 
    162   def GetAll(self):
    163     """Get the set of all test results."""
    164     return self._results.copy()
    165 
    166   def _GetType(self, test_type):
    167     """Get the set of test results with the given test type."""
    168     return set(t for t in self._results if t.GetType() == test_type)
    169 
    170   def GetPass(self):
    171     """Get the set of all passed test results."""
    172     return self._GetType(ResultType.PASS)
    173 
    174   def GetSkip(self):
    175     """Get the set of all skipped test results."""
    176     return self._GetType(ResultType.SKIP)
    177 
    178   def GetFail(self):
    179     """Get the set of all failed test results."""
    180     return self._GetType(ResultType.FAIL)
    181 
    182   def GetCrash(self):
    183     """Get the set of all crashed test results."""
    184     return self._GetType(ResultType.CRASH)
    185 
    186   def GetTimeout(self):
    187     """Get the set of all timed out test results."""
    188     return self._GetType(ResultType.TIMEOUT)
    189 
    190   def GetUnknown(self):
    191     """Get the set of all unknown test results."""
    192     return self._GetType(ResultType.UNKNOWN)
    193 
    194   def GetNotPass(self):
    195     """Get the set of all non-passed test results."""
    196     return self.GetAll() - self.GetPass()
    197 
    198   def DidRunPass(self):
    199     """Return whether the test run was successful."""
    200     return not (self.GetNotPass() - self.GetSkip())
    201 
    202