Home | History | Annotate | Download | only in scripts 
      1 #!/usr/bin/python
      2 # Copyright 2016 The Chromium OS Authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 # pylint: disable-msg=W0311
      7 
      8 import argparse
      9 import json
     10 import os
     11 
     12 
     13 gpu_list = [
     14     #'pinetrail',
     15     'sandybridge',
     16     'ivybridge',
     17     'baytrail',
     18     'haswell',
     19     'broadwell',
     20     'braswell',
     21     'skylake',
     22     'broxton',
     23     'mali-t604',
     24     'mali-t628',
     25     'mali-t760',
     26     'mali-t860',
     27     'rogue',
     28     'tegra',
     29 ]
     30 
     31 _PROBLEM_STATUS = ['Fail', 'Flaky']
     32 _UNKNOWN_STATUS = ['NotSupported', 'Skipped', 'Unknown', None]
     33 
     34 status_dict = {
     35     'Fail': 'FAIL ',
     36     'Flaky': 'flaky',
     37     'Pass': '  +  ',
     38     'NotSupported': ' --- ',
     39     'Skipped': ' === ',
     40     'QualityWarning': 'qw   ',
     41     'CompatibilityWarning': 'cw   ',
     42     'Unknown': ' ??? ',
     43     None: ' n/a ',
     44 }
     45 
     46 def load_expectation_dict(json_file):
     47   data = {}
     48   if os.path.isfile(json_file):
     49     with open(json_file, 'r') as f:
     50       text = f.read()
     51       data = json.loads(text)
     52   return data
     53 
     54 
     55 def load_expectations(json_file):
     56   data = load_expectation_dict(json_file)
     57   expectations = {}
     58   # Convert from dictionary of lists to dictionary of sets.
     59   for key in data:
     60     expectations[key] = set(data[key])
     61   return expectations
     62 
     63 
     64 def get_problem_count(dict, gpu):
     65   if gpu in dict:
     66     if not dict[gpu]:
     67       return None
     68     count = 0
     69     for status in dict[gpu]:
     70       if status not in _UNKNOWN_STATUS:
     71         count = count + len((dict[gpu])[status])
     72     # If every result has an unknown status then don't return a count.
     73     if count < 1:
     74       return None
     75     count = 0
     76     # Return counts of truly problematic statuses.
     77     for status in _PROBLEM_STATUS:
     78       if status in dict[gpu]:
     79         count = count + len((dict[gpu])[status])
     80   else:
     81     print 'Warning: %s not found in dict!' % gpu
     82   return count
     83 
     84 
     85 def get_problem_tests(dict):
     86   tests = set([])
     87   for gpu in dict:
     88     for status in _PROBLEM_STATUS:
     89       if status in dict[gpu]:
     90         tests = tests.union((dict[gpu])[status])
     91   return sorted(list(tests))
     92 
     93 
     94 def get_test_result(dict, test):
     95   for key in dict:
     96     if test in dict[key]:
     97       return key
     98   return None
     99 
    100 
    101 argparser = argparse.ArgumentParser(
    102     description='Create a matrix of failing tests per GPU.')
    103 argparser.add_argument('interface',
    104                        default='gles2',
    105                        help='Interface for matrix (gles2, gles3, gles31).')
    106 args = argparser.parse_args()
    107 status = '%s-master.txt.json' % args.interface
    108 
    109 dict = {}
    110 for gpu in gpu_list:
    111   filename = 'expectations/%s/%s' % (gpu, status)
    112   dict[gpu] = load_expectations(filename)
    113 
    114 tests = get_problem_tests(dict)
    115 
    116 print 'Legend:'
    117 for key in status_dict:
    118   print '%s  -->  %s' % (status_dict[key], key)
    119 print
    120 
    121 offset = ''
    122 for gpu in gpu_list:
    123   print '%s%s' % (offset, gpu)
    124   offset = '%s    |    ' % offset
    125 print offset
    126 
    127 text_count = ''
    128 text_del = ''
    129 for gpu in gpu_list:
    130   problem_count = get_problem_count(dict, gpu)
    131   if problem_count is None:
    132     text_count = '%s  %s  ' % (text_count, status_dict[problem_count])
    133   else:
    134     text_count = '%s%5d    ' % (text_count, problem_count)
    135   text_del = '%s=========' % text_del
    136 text_count = '%s  Total failure count (Fail + Flaky)' % text_count
    137 print text_del
    138 print text_count
    139 print text_del
    140 
    141 for test in tests:
    142   text = ''
    143   for gpu in gpu_list:
    144     result = get_test_result(dict[gpu], test)
    145     status = status_dict[result]
    146     text = '%s  %s  ' % (text, status)
    147   text = '%s  %s' % (text, test)
    148   print text
    149 
    150 print text_del
    151 print '%s repeated' % text_count
    152 print text_del
    153 
    154