1 # -*- coding: utf-8 -*- 2 3 #------------------------------------------------------------------------- 4 # drawElements Quality Program utilities 5 # -------------------------------------- 6 # 7 # Copyright 2015 The Android Open Source Project 8 # 9 # Licensed under the Apache License, Version 2.0 (the "License"); 10 # you may not use this file except in compliance with the License. 11 # You may obtain a copy of the License at 12 # 13 # http://www.apache.org/licenses/LICENSE-2.0 14 # 15 # Unless required by applicable law or agreed to in writing, software 16 # distributed under the License is distributed on an "AS IS" BASIS, 17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 # See the License for the specific language governing permissions and 19 # limitations under the License. 20 # 21 #------------------------------------------------------------------------- 22 23 import sys 24 import random 25 import string 26 import subprocess 27 from optparse import OptionParser 28 29 def all (results, predicate): 30 for result in results: 31 if not predicate(result): 32 return False 33 return True 34 35 def any (results, predicate): 36 for result in results: 37 if predicate(result): 38 return True 39 return False 40 41 class FilterRule: 42 def __init__ (self, name, description, filters): 43 self.name = name 44 self.description = description 45 self.filters = filters 46 47 class TestCaseResult: 48 def __init__ (self, name, results): 49 self.name = name 50 self.results = results 51 52 class Group: 53 def __init__ (self, name): 54 self.name = name 55 self.cases = [] 56 57 def readCaseList (filename): 58 f = open(filename, 'rb') 59 cases = [] 60 for line in f: 61 if line[:6] == "TEST: ": 62 case = line[6:].strip() 63 if len(case) > 0: 64 cases.append(case) 65 return cases 66 67 def toResultList (caselist): 68 results = [] 69 for case in caselist: 70 results.append(TestCaseResult(case, [])) 71 return results 72 73 def addResultsToCaseList (caselist, results): 74 resultMap = {} 75 caseListRes = toResultList(caselist) 76 77 for res in caseListRes: 78 resultMap[res.name] = res 79 80 for result in results: 81 if result.name in resultMap: 82 resultMap[result.name].results += result.results 83 84 return caseListRes 85 86 def readTestResults (filename): 87 f = open(filename, 'rb') 88 csvData = f.read() 89 csvLines = csvData.splitlines() 90 results = [] 91 92 f.close() 93 94 for line in csvLines[1:]: 95 args = line.split(',') 96 if len(args) == 1: 97 continue # Ignore 98 99 results.append(TestCaseResult(args[0], args[1:])) 100 101 if len(results) == 0: 102 raise Exception("Empty result list") 103 104 # Sanity check for results 105 numResultItems = len(results[0].results) 106 seenResults = set() 107 for result in results: 108 if result.name in seenResults: 109 raise Exception("Duplicate result row for test case '%s'" % result.name) 110 if len(result.results) != numResultItems: 111 raise Exception("Found %d results for test case '%s', expected %d" % (len(result.results), result.name, numResultItems)) 112 seenResults.add(result.name) 113 114 return results 115 116 def readGroupList (filename): 117 f = open(filename, 'rb') 118 groups = [] 119 for line in f: 120 group = line.strip() 121 if group != "": 122 groups.append(group) 123 return groups 124 125 def createGroups (results, groupNames): 126 groups = [] 127 matched = set() 128 129 for groupName in groupNames: 130 group = Group(groupName) 131 groups.append(group) 132 133 prefix = groupName + "." 134 prefixLen = len(prefix) 135 for case in results: 136 if case.name[:prefixLen] == prefix: 137 if case in matched: 138 die("Case '%s' matched by multiple groups (when processing '%s')" % (case.name, group.name)) 139 group.cases.append(case) 140 matched.add(case) 141 142 return groups 143 144 def createLeafGroups (results): 145 groups = [] 146 groupMap = {} 147 148 for case in results: 149 parts = case.name.split('.') 150 groupName = string.join(parts[:-1], ".") 151 152 if not groupName in groupMap: 153 group = Group(groupName) 154 groups.append(group) 155 groupMap[groupName] = group 156 else: 157 group = groupMap[groupName] 158 159 group.cases.append(case) 160 161 return groups 162 163 def filterList (results, condition): 164 filtered = [] 165 for case in results: 166 if condition(case.results): 167 filtered.append(case) 168 return filtered 169 170 def getFilter (list, name): 171 for filter in list: 172 if filter.name == name: 173 return filter 174 return None 175 176 def getNumCasesInGroups (groups): 177 numCases = 0 178 for group in groups: 179 numCases += len(group.cases) 180 return numCases 181 182 def getCasesInSet (results, caseSet): 183 filtered = [] 184 for case in results: 185 if case in caseSet: 186 filtered.append(case) 187 return filtered 188 189 def selectCasesInGroups (results, groups): 190 casesInGroups = set() 191 for group in groups: 192 for case in group.cases: 193 casesInGroups.add(case) 194 return getCasesInSet(results, casesInGroups) 195 196 def selectRandomSubset (results, groups, limit, seed): 197 selectedCases = set() 198 numSelect = min(limit, getNumCasesInGroups(groups)) 199 200 random.seed(seed) 201 random.shuffle(groups) 202 203 groupNdx = 0 204 while len(selectedCases) < numSelect: 205 group = groups[groupNdx] 206 if len(group.cases) == 0: 207 del groups[groupNdx] 208 if groupNdx == len(groups): 209 groupNdx -= 1 210 continue # Try next 211 212 selected = random.choice(group.cases) 213 selectedCases.add(selected) 214 group.cases.remove(selected) 215 216 groupNdx = (groupNdx + 1) % len(groups) 217 218 return getCasesInSet(results, selectedCases) 219 220 def die (msg): 221 print msg 222 sys.exit(-1) 223 224 # Named filter lists 225 FILTER_RULES = [ 226 FilterRule("all", "No filtering", []), 227 FilterRule("all-pass", "All results must be 'Pass'", [lambda l: all(l, lambda r: r == 'Pass')]), 228 FilterRule("any-pass", "Any of results is 'Pass'", [lambda l: any(l, lambda r: r == 'Pass')]), 229 FilterRule("any-fail", "Any of results is not 'Pass' or 'NotSupported'", [lambda l: not all(l, lambda r: r == 'Pass' or r == 'NotSupported')]), 230 FilterRule("prev-failing", "Any except last result is failure", [lambda l: l[-1] == 'Pass' and not all(l[:-1], lambda r: r == 'Pass')]), 231 FilterRule("prev-passing", "Any except last result is 'Pass'", [lambda l: l[-1] != 'Pass' and any(l[:-1], lambda r: r == 'Pass')]) 232 ] 233 234 if __name__ == "__main__": 235 parser = OptionParser(usage = "usage: %prog [options] [caselist] [result csv file]") 236 parser.add_option("-f", "--filter", dest="filter", default="all", help="filter rule name") 237 parser.add_option("-l", "--list", action="store_true", dest="list", default=False, help="list available rules") 238 parser.add_option("-n", "--num", dest="limit", default=0, help="limit number of cases") 239 parser.add_option("-s", "--seed", dest="seed", default=0, help="use selected seed for random selection") 240 parser.add_option("-g", "--groups", dest="groups_file", default=None, help="select cases based on group list file") 241 242 (options, args) = parser.parse_args() 243 244 if options.list: 245 print "Available filter rules:" 246 for filter in FILTER_RULES: 247 print " %s: %s" % (filter.name, filter.description) 248 sys.exit(0) 249 250 if len(args) == 0: 251 die("No input files specified") 252 elif len(args) > 2: 253 die("Too many arguments") 254 255 # Fetch filter 256 filter = getFilter(FILTER_RULES, options.filter) 257 if filter == None: 258 die("Unknown filter '%s'" % options.filter) 259 260 # Read case list 261 caselist = readCaseList(args[0]) 262 if len(args) > 1: 263 results = readTestResults(args[1]) 264 results = addResultsToCaseList(caselist, results) 265 else: 266 results = toResultList(caselist) 267 268 # Execute filters for results 269 for rule in filter.filters: 270 results = filterList(results, rule) 271 272 if options.limit != 0: 273 if options.groups_file != None: 274 groups = createGroups(results, readGroupList(options.groups_file)) 275 else: 276 groups = createLeafGroups(results) 277 results = selectRandomSubset(results, groups, int(options.limit), int(options.seed)) 278 elif options.groups_file != None: 279 groups = createGroups(results, readGroupList(options.groups_file)) 280 results = selectCasesInGroups(results, groups) 281 282 # Print test set 283 for result in results: 284 print result.name 285
