Home | History | Annotate | Download | only in site_utils 
      1 # Copyright 2018 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 """Common file shared by test_push of autotest and skylab.
      6 
      7 autotest: site_utils/test_push.py
      8 skylab: venv/skylab_staging/test_push.py
      9 """
     10 
     11 import collections
     12 import re
     13 
     14 # Dictionary of test results keyed by test name regular expression.
     15 EXPECTED_TEST_RESULTS = {'^SERVER_JOB$':                 'GOOD',
     16                          # This is related to dummy_Fail/control.dependency.
     17                          'dummy_Fail.dependency$':       'TEST_NA',
     18                          'login_LoginSuccess.*':         'GOOD',
     19                          'provision_AutoUpdate.double':  'GOOD',
     20                          'dummy_Pass.*':                 'GOOD',
     21                          'dummy_Fail.Fail$':             'FAIL',
     22                          'dummy_Fail.Error$':            'ERROR',
     23                          'dummy_Fail.Warn$':             'WARN',
     24                          'dummy_Fail.NAError$':          'TEST_NA',
     25                          'dummy_Fail.Crash$':            'GOOD',
     26                          'autotest_SyncCount$':          'GOOD',
     27                          }
     28 
     29 EXPECTED_TEST_RESULTS_DUMMY = {'^SERVER_JOB$':       'GOOD',
     30                                'dummy_Pass.*':       'GOOD',
     31                                'dummy_Fail.Fail':    'FAIL',
     32                                'dummy_Fail.Warn':    'WARN',
     33                                'dummy_Fail.Crash':   'GOOD',
     34                                'dummy_Fail.Error':   'ERROR',
     35                                'dummy_Fail.NAError': 'TEST_NA',}
     36 
     37 EXPECTED_TEST_RESULTS_POWERWASH = {'platform_Powerwash': 'GOOD',
     38                                    'SERVER_JOB':         'GOOD'}
     39 
     40 _TestPushErrors = collections.namedtuple(
     41         '_TestPushErrors',
     42         [
     43                 'mismatch_errors',
     44                 'unknown_tests',
     45                 'missing_tests',
     46         ]
     47 )
     48 
     49 
     50 def summarize_push(test_views, expected_results, ignored_tests=[]):
     51     """Summarize the test push errors."""
     52     test_push_errors = _match_test_results(test_views, expected_results,
     53                                            ignored_tests)
     54     return _generate_push_summary(test_push_errors)
     55 
     56 
     57 def _match_test_results(test_views, expected_results, ignored_tests):
     58     """Match test results with expected results.
     59 
     60     @param test_views: A dictionary of test status keyed by test name, e.g.,
     61              {'dummy_Fail.Error': 'ERROR', 'dummy_Fail.NAError': 'TEST_NA'}
     62     @param expected_results: A dictionary of test name to expected test result.
     63 
     64     @return: A _TestPushErrors tuple.
     65     """
     66     mismatch_errors = []
     67     unknown_tests = []
     68     found_keys = set()
     69     for test_name, test_status in test_views.iteritems():
     70         test_found = False
     71         for test_name_pattern, expected_result in expected_results.items():
     72             if re.search(test_name_pattern, test_name):
     73                 test_found = True
     74                 found_keys.add(test_name_pattern)
     75                 if (expected_result != test_status and
     76                     _is_significant(test_name, ignored_tests)):
     77                     error = ('%s Expected: [%s], Actual: [%s]' %
     78                              (test_name, expected_result, test_status))
     79                     mismatch_errors.append(error)
     80 
     81         if not test_found and _is_significant(test_name, ignored_tests):
     82             unknown_tests.append(test_name)
     83 
     84     missing_tests = set(expected_results.keys()) - found_keys
     85     missing_tests = [t for t in missing_tests
     86                      if _is_significant(t, ignored_tests)]
     87     return _TestPushErrors(mismatch_errors=mismatch_errors,
     88                            unknown_tests=unknown_tests,
     89                            missing_tests=missing_tests)
     90 
     91 
     92 def _is_significant(test, ignored_tests_patterns):
     93     return all([test not in m for m in ignored_tests_patterns])
     94 
     95 
     96 def _generate_push_summary(test_push_errors):
     97     """Generate a list of summary based on the test_push results."""
     98     summary = []
     99     if test_push_errors.mismatch_errors:
    100         summary.append(('Results of %d test(s) do not match expected '
    101                         'values:') % len(test_push_errors.mismatch_errors))
    102         summary.extend(test_push_errors.mismatch_errors)
    103         summary.append('\n')
    104 
    105     if test_push_errors.unknown_tests:
    106         summary.append('%d test(s) are not expected to be run:' %
    107                        len(test_push_errors.unknown_tests))
    108         summary.extend(test_push_errors.unknown_tests)
    109         summary.append('\n')
    110 
    111     if test_push_errors.missing_tests:
    112         summary.append('%d test(s) are missing from the results:' %
    113                        len(test_push_errors.missing_tests))
    114         summary.extend(test_push_errors.missing_tests)
    115         summary.append('\n')
    116 
    117     return summary
    118