1 # -*- coding: utf-8 -*- 2 3 import sys 4 import random 5 import string 6 import subprocess 7 from optparse import OptionParser 8 9 def all (results, predicate): 10 for result in results: 11 if not predicate(result): 12 return False 13 return True 14 15 def any (results, predicate): 16 for result in results: 17 if predicate(result): 18 return True 19 return False 20 21 class FilterRule: 22 def __init__ (self, name, description, filters): 23 self.name = name 24 self.description = description 25 self.filters = filters 26 27 class TestCaseResult: 28 def __init__ (self, name, results): 29 self.name = name 30 self.results = results 31 32 class Group: 33 def __init__ (self, name): 34 self.name = name 35 self.cases = [] 36 37 def readCaseList (filename): 38 f = open(filename, 'rb') 39 cases = [] 40 for line in f: 41 if line[:6] == "TEST: ": 42 case = line[6:].strip() 43 if len(case) > 0: 44 cases.append(case) 45 return cases 46 47 def toResultList (caselist): 48 results = [] 49 for case in caselist: 50 results.append(TestCaseResult(case, [])) 51 return results 52 53 def addResultsToCaseList (caselist, results): 54 resultMap = {} 55 caseListRes = toResultList(caselist) 56 57 for res in caseListRes: 58 resultMap[res.name] = res 59 60 for result in results: 61 if result.name in resultMap: 62 resultMap[result.name].results += result.results 63 64 return caseListRes 65 66 def readTestResults (filename): 67 f = open(filename, 'rb') 68 csvData = f.read() 69 csvLines = csvData.splitlines() 70 results = [] 71 72 f.close() 73 74 for line in csvLines[1:]: 75 args = line.split(',') 76 if len(args) == 1: 77 continue # Ignore 78 79 results.append(TestCaseResult(args[0], args[1:])) 80 81 if len(results) == 0: 82 raise Exception("Empty result list") 83 84 # Sanity check for results 85 numResultItems = len(results[0].results) 86 seenResults = set() 87 for result in results: 88 if result.name in seenResults: 89 raise Exception("Duplicate result row for test case '%s'" % result.name) 90 if len(result.results) != numResultItems: 91 raise Exception("Found %d results for test case '%s', expected %d" % (len(result.results), result.name, numResultItems)) 92 seenResults.add(result.name) 93 94 return results 95 96 def readGroupList (filename): 97 f = open(filename, 'rb') 98 groups = [] 99 for line in f: 100 group = line.strip() 101 if group != "": 102 groups.append(group) 103 return groups 104 105 def createGroups (results, groupNames): 106 groups = [] 107 matched = set() 108 109 for groupName in groupNames: 110 group = Group(groupName) 111 groups.append(group) 112 113 prefix = groupName + "." 114 prefixLen = len(prefix) 115 for case in results: 116 if case.name[:prefixLen] == prefix: 117 if case in matched: 118 die("Case '%s' matched by multiple groups (when processing '%s')" % (case.name, group.name)) 119 group.cases.append(case) 120 matched.add(case) 121 122 return groups 123 124 def createLeafGroups (results): 125 groups = [] 126 groupMap = {} 127 128 for case in results: 129 parts = case.name.split('.') 130 groupName = string.join(parts[:-1], ".") 131 132 if not groupName in groupMap: 133 group = Group(groupName) 134 groups.append(group) 135 groupMap[groupName] = group 136 else: 137 group = groupMap[groupName] 138 139 group.cases.append(case) 140 141 return groups 142 143 def filterList (results, condition): 144 filtered = [] 145 for case in results: 146 if condition(case.results): 147 filtered.append(case) 148 return filtered 149 150 def getFilter (list, name): 151 for filter in list: 152 if filter.name == name: 153 return filter 154 return None 155 156 def getNumCasesInGroups (groups): 157 numCases = 0 158 for group in groups: 159 numCases += len(group.cases) 160 return numCases 161 162 def getCasesInSet (results, caseSet): 163 filtered = [] 164 for case in results: 165 if case in caseSet: 166 filtered.append(case) 167 return filtered 168 169 def selectCasesInGroups (results, groups): 170 casesInGroups = set() 171 for group in groups: 172 for case in group.cases: 173 casesInGroups.add(case) 174 return getCasesInSet(results, casesInGroups) 175 176 def selectRandomSubset (results, groups, limit, seed): 177 selectedCases = set() 178 numSelect = min(limit, getNumCasesInGroups(groups)) 179 180 random.seed(seed) 181 random.shuffle(groups) 182 183 groupNdx = 0 184 while len(selectedCases) < numSelect: 185 group = groups[groupNdx] 186 if len(group.cases) == 0: 187 del groups[groupNdx] 188 if groupNdx == len(groups): 189 groupNdx -= 1 190 continue # Try next 191 192 selected = random.choice(group.cases) 193 selectedCases.add(selected) 194 group.cases.remove(selected) 195 196 groupNdx = (groupNdx + 1) % len(groups) 197 198 return getCasesInSet(results, selectedCases) 199 200 def die (msg): 201 print msg 202 sys.exit(-1) 203 204 # Named filter lists 205 FILTER_RULES = [ 206 FilterRule("all", "No filtering", []), 207 FilterRule("all-pass", "All results must be 'Pass'", [lambda l: all(l, lambda r: r == 'Pass')]), 208 FilterRule("any-pass", "Any of results is 'Pass'", [lambda l: any(l, lambda r: r == 'Pass')]), 209 FilterRule("any-fail", "Any of results is not 'Pass' or 'NotSupported'", [lambda l: not all(l, lambda r: r == 'Pass' or r == 'NotSupported')]), 210 FilterRule("prev-failing", "Any except last result is failure", [lambda l: l[-1] == 'Pass' and not all(l[:-1], lambda r: r == 'Pass')]), 211 FilterRule("prev-passing", "Any except last result is 'Pass'", [lambda l: l[-1] != 'Pass' and any(l[:-1], lambda r: r == 'Pass')]) 212 ] 213 214 if __name__ == "__main__": 215 parser = OptionParser(usage = "usage: %prog [options] [caselist] [result csv file]") 216 parser.add_option("-f", "--filter", dest="filter", default="all", help="filter rule name") 217 parser.add_option("-l", "--list", action="store_true", dest="list", default=False, help="list available rules") 218 parser.add_option("-n", "--num", dest="limit", default=0, help="limit number of cases") 219 parser.add_option("-s", "--seed", dest="seed", default=0, help="use selected seed for random selection") 220 parser.add_option("-g", "--groups", dest="groups_file", default=None, help="select cases based on group list file") 221 222 (options, args) = parser.parse_args() 223 224 if options.list: 225 print "Available filter rules:" 226 for filter in FILTER_RULES: 227 print " %s: %s" % (filter.name, filter.description) 228 sys.exit(0) 229 230 if len(args) == 0: 231 die("No input files specified") 232 elif len(args) > 2: 233 die("Too many arguments") 234 235 # Fetch filter 236 filter = getFilter(FILTER_RULES, options.filter) 237 if filter == None: 238 die("Unknown filter '%s'" % options.filter) 239 240 # Read case list 241 caselist = readCaseList(args[0]) 242 if len(args) > 1: 243 results = readTestResults(args[1]) 244 results = addResultsToCaseList(caselist, results) 245 else: 246 results = toResultList(caselist) 247 248 # Execute filters for results 249 for rule in filter.filters: 250 results = filterList(results, rule) 251 252 if options.limit != 0: 253 if options.groups_file != None: 254 groups = createGroups(results, readGroupList(options.groups_file)) 255 else: 256 groups = createLeafGroups(results) 257 results = selectRandomSubset(results, groups, int(options.limit), int(options.seed)) 258 elif options.groups_file != None: 259 groups = createGroups(results, readGroupList(options.groups_file)) 260 results = selectCasesInGroups(results, groups) 261 262 # Print test set 263 for result in results: 264 print result.name 265
