Home | History | Annotate | Download | only in scripts 
      1 #!/usr/bin/python
      2 # Copyright 2016 The Chromium OS Authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 # pylint: disable-msg=W0311
      7 
      8 import argparse
      9 import json
     10 import os
     11 
     12 
     13 gpu_list = [
     14     #'pinetrail',
     15     'sandybridge',
     16     'ivybridge',
     17     'baytrail',
     18     'haswell',
     19     'broadwell',
     20     'braswell',
     21     'skylake',
     22     'mali-t604',
     23     'mali-t628',
     24     'mali-t760',
     25     'rogue',
     26     'tegra',
     27 ]
     28 
     29 _PROBLEM_STATUS = ['Fail', 'Flaky']
     30 _UNKNOWN_STATUS = ['NotSupported', 'Skipped', 'Unknown', None]
     31 
     32 status_dict = {
     33     'Fail': 'FAIL ',
     34     'Flaky': 'flaky',
     35     'Pass': '  +  ',
     36     'NotSupported': ' --- ',
     37     'Skipped': ' === ',
     38     'QualityWarning': 'qw   ',
     39     'CompatibilityWarning': 'cw   ',
     40     'Unknown': ' ??? ',
     41     None: ' n/a ',
     42 }
     43 
     44 def load_expectation_dict(json_file):
     45   data = {}
     46   if os.path.isfile(json_file):
     47     with open(json_file, 'r') as f:
     48       text = f.read()
     49       data = json.loads(text)
     50   return data
     51 
     52 
     53 def load_expectations(json_file):
     54   data = load_expectation_dict(json_file)
     55   expectations = {}
     56   # Convert from dictionary of lists to dictionary of sets.
     57   for key in data:
     58     expectations[key] = set(data[key])
     59   return expectations
     60 
     61 
     62 def get_problem_count(dict, gpu):
     63   if gpu in dict:
     64     if not dict[gpu]:
     65       return None
     66     count = 0
     67     for status in dict[gpu]:
     68       if status not in _UNKNOWN_STATUS:
     69         count = count + len((dict[gpu])[status])
     70     # If every result has an unknown status then don't return a count.
     71     if count < 1:
     72       return None
     73     count = 0
     74     # Return counts of truly problematic statuses.
     75     for status in _PROBLEM_STATUS:
     76       if status in dict[gpu]:
     77         count = count + len((dict[gpu])[status])
     78   else:
     79     print 'Warning: %s not found in dict!' % gpu
     80   return count
     81 
     82 
     83 def get_problem_tests(dict):
     84   tests = set([])
     85   for gpu in dict:
     86     for status in _PROBLEM_STATUS:
     87       if status in dict[gpu]:
     88         tests = tests.union((dict[gpu])[status])
     89   return sorted(list(tests))
     90 
     91 
     92 def get_test_result(dict, test):
     93   for key in dict:
     94     if test in dict[key]:
     95       return key
     96   return None
     97 
     98 
     99 argparser = argparse.ArgumentParser(
    100     description='Create a matrix of failing tests per GPU.')
    101 argparser.add_argument('interface',
    102                        default='gles2',
    103                        help='Interface for matrix (gles2, gles3, gles31).')
    104 args = argparser.parse_args()
    105 status = '%s-master.json' % args.interface
    106 
    107 dict = {}
    108 for gpu in gpu_list:
    109   filename = 'expectations/%s/%s' % (gpu, status)
    110   dict[gpu] = load_expectations(filename)
    111 
    112 tests = get_problem_tests(dict)
    113 
    114 print 'Legend:'
    115 for key in status_dict:
    116   print '%s  -->  %s' % (status_dict[key], key)
    117 print
    118 
    119 offset = ''
    120 for gpu in gpu_list:
    121   print '%s%s' % (offset, gpu)
    122   offset = '%s    |    ' % offset
    123 print offset
    124 
    125 text_count = ''
    126 text_del = ''
    127 for gpu in gpu_list:
    128   problem_count = get_problem_count(dict, gpu)
    129   if problem_count is None:
    130     text_count = '%s  %s  ' % (text_count, status_dict[problem_count])
    131   else:
    132     text_count = '%s%5d    ' % (text_count, problem_count)
    133   text_del = '%s=========' % text_del
    134 text_count = '%s  Total failure count (Fail + Flaky)' % text_count
    135 print text_del
    136 print text_count
    137 print text_del
    138 
    139 for test in tests:
    140   text = ''
    141   for gpu in gpu_list:
    142     result = get_test_result(dict[gpu], test)
    143     status = status_dict[result]
    144     text = '%s  %s  ' % (text, status)
    145   text = '%s  %s' % (text, test)
    146   print text
    147 
    148 print text_del
    149 print '%s repeated' % text_count
    150 print text_del
    151 
    152