1 #!/usr/bin/python 2 # Copyright 2016 The Chromium OS Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 # pylint: disable-msg=W0311 7 8 import argparse 9 import json 10 import os 11 12 13 gpu_list = [ 14 #'pinetrail', 15 'sandybridge', 16 'ivybridge', 17 'baytrail', 18 'haswell', 19 'broadwell', 20 'braswell', 21 'skylake', 22 'broxton', 23 'mali-t604', 24 'mali-t628', 25 'mali-t760', 26 'mali-t860', 27 'rogue', 28 'tegra', 29 ] 30 31 _PROBLEM_STATUS = ['Fail', 'Flaky'] 32 _UNKNOWN_STATUS = ['NotSupported', 'Skipped', 'Unknown', None] 33 34 status_dict = { 35 'Fail': 'FAIL ', 36 'Flaky': 'flaky', 37 'Pass': ' + ', 38 'NotSupported': ' --- ', 39 'Skipped': ' === ', 40 'QualityWarning': 'qw ', 41 'CompatibilityWarning': 'cw ', 42 'Unknown': ' ??? ', 43 None: ' n/a ', 44 } 45 46 def load_expectation_dict(json_file): 47 data = {} 48 if os.path.isfile(json_file): 49 with open(json_file, 'r') as f: 50 text = f.read() 51 data = json.loads(text) 52 return data 53 54 55 def load_expectations(json_file): 56 data = load_expectation_dict(json_file) 57 expectations = {} 58 # Convert from dictionary of lists to dictionary of sets. 59 for key in data: 60 expectations[key] = set(data[key]) 61 return expectations 62 63 64 def get_problem_count(dict, gpu): 65 if gpu in dict: 66 if not dict[gpu]: 67 return None 68 count = 0 69 for status in dict[gpu]: 70 if status not in _UNKNOWN_STATUS: 71 count = count + len((dict[gpu])[status]) 72 # If every result has an unknown status then don't return a count. 73 if count < 1: 74 return None 75 count = 0 76 # Return counts of truly problematic statuses. 77 for status in _PROBLEM_STATUS: 78 if status in dict[gpu]: 79 count = count + len((dict[gpu])[status]) 80 else: 81 print 'Warning: %s not found in dict!' % gpu 82 return count 83 84 85 def get_problem_tests(dict): 86 tests = set([]) 87 for gpu in dict: 88 for status in _PROBLEM_STATUS: 89 if status in dict[gpu]: 90 tests = tests.union((dict[gpu])[status]) 91 return sorted(list(tests)) 92 93 94 def get_test_result(dict, test): 95 for key in dict: 96 if test in dict[key]: 97 return key 98 return None 99 100 101 argparser = argparse.ArgumentParser( 102 description='Create a matrix of failing tests per GPU.') 103 argparser.add_argument('interface', 104 default='gles2', 105 help='Interface for matrix (gles2, gles3, gles31).') 106 args = argparser.parse_args() 107 status = '%s-master.txt.json' % args.interface 108 109 dict = {} 110 for gpu in gpu_list: 111 filename = 'expectations/%s/%s' % (gpu, status) 112 dict[gpu] = load_expectations(filename) 113 114 tests = get_problem_tests(dict) 115 116 print 'Legend:' 117 for key in status_dict: 118 print '%s --> %s' % (status_dict[key], key) 119 print 120 121 offset = '' 122 for gpu in gpu_list: 123 print '%s%s' % (offset, gpu) 124 offset = '%s | ' % offset 125 print offset 126 127 text_count = '' 128 text_del = '' 129 for gpu in gpu_list: 130 problem_count = get_problem_count(dict, gpu) 131 if problem_count is None: 132 text_count = '%s %s ' % (text_count, status_dict[problem_count]) 133 else: 134 text_count = '%s%5d ' % (text_count, problem_count) 135 text_del = '%s=========' % text_del 136 text_count = '%s Total failure count (Fail + Flaky)' % text_count 137 print text_del 138 print text_count 139 print text_del 140 141 for test in tests: 142 text = '' 143 for gpu in gpu_list: 144 result = get_test_result(dict[gpu], test) 145 status = status_dict[result] 146 text = '%s %s ' % (text, status) 147 text = '%s %s' % (text, test) 148 print text 149 150 print text_del 151 print '%s repeated' % text_count 152 print text_del 153 154