1 # -*- coding: utf-8 -*- 2 3 #------------------------------------------------------------------------- 4 # drawElements Quality Program utilities 5 # -------------------------------------- 6 # 7 # Copyright 2016 The Android Open Source Project 8 # 9 # Licensed under the Apache License, Version 2.0 (the "License"); 10 # you may not use this file except in compliance with the License. 11 # You may obtain a copy of the License at 12 # 13 # http://www.apache.org/licenses/LICENSE-2.0 14 # 15 # Unless required by applicable law or agreed to in writing, software 16 # distributed under the License is distributed on an "AS IS" BASIS, 17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 # See the License for the specific language governing permissions and 19 # limitations under the License. 20 # 21 #------------------------------------------------------------------------- 22 23 from build.common import * 24 from build.config import ANY_GENERATOR 25 from build.build import build 26 from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET 27 from fnmatch import fnmatch 28 from copy import copy 29 30 import xml.etree.cElementTree as ElementTree 31 import xml.dom.minidom as minidom 32 33 APK_NAME = "com.drawelements.deqp.apk" 34 35 GENERATED_FILE_WARNING = """ 36 This file has been automatically generated. Edit with caution. 37 """ 38 39 class Project: 40 def __init__ (self, path, copyright = None): 41 self.path = path 42 self.copyright = copyright 43 44 class Configuration: 45 def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, runtime = None): 46 self.name = name 47 self.glconfig = glconfig 48 self.rotation = rotation 49 self.surfacetype = surfacetype 50 self.filters = filters 51 self.expectedRuntime = runtime 52 53 class Package: 54 def __init__ (self, module, configurations): 55 self.module = module 56 self.configurations = configurations 57 58 class Mustpass: 59 def __init__ (self, project, version, packages): 60 self.project = project 61 self.version = version 62 self.packages = packages 63 64 class Filter: 65 TYPE_INCLUDE = 0 66 TYPE_EXCLUDE = 1 67 68 def __init__ (self, type, filename): 69 self.type = type 70 self.filename = filename 71 72 class TestRoot: 73 def __init__ (self): 74 self.children = [] 75 76 class TestGroup: 77 def __init__ (self, name): 78 self.name = name 79 self.children = [] 80 81 class TestCase: 82 def __init__ (self, name): 83 self.name = name 84 self.configurations = [] 85 86 class GLESVersion: 87 def __init__(self, major, minor): 88 self.major = major 89 self.minor = minor 90 91 def encode (self): 92 return (self.major << 16) | (self.minor) 93 94 def getModuleGLESVersion (module): 95 versions = { 96 'dEQP-EGL': GLESVersion(2,0), 97 'dEQP-GLES2': GLESVersion(2,0), 98 'dEQP-GLES3': GLESVersion(3,0), 99 'dEQP-GLES31': GLESVersion(3,1) 100 } 101 return versions[module.name] if module.name in versions else None 102 103 def getSrcDir (mustpass): 104 return os.path.join(mustpass.project.path, mustpass.version, "src") 105 106 def getTmpDir (mustpass): 107 return os.path.join(mustpass.project.path, mustpass.version, "tmp") 108 109 def getModuleShorthand (module): 110 assert module.name[:5] == "dEQP-" 111 return module.name[5:].lower() 112 113 def getCaseListFileName (package, configuration): 114 return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name) 115 116 def getDstCaseListPath (mustpass, package, configuration): 117 return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration)) 118 119 def getCTSPackageName (package): 120 return "com.drawelements.deqp." + getModuleShorthand(package.module) 121 122 def getCommandLine (config): 123 cmdLine = "" 124 125 if config.glconfig != None: 126 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig 127 128 if config.rotation != None: 129 cmdLine += "--deqp-screen-rotation=%s " % config.rotation 130 131 if config.surfacetype != None: 132 cmdLine += "--deqp-surface-type=%s " % config.surfacetype 133 134 cmdLine += "--deqp-watchdog=enable" 135 136 return cmdLine 137 138 def readCaseList (filename): 139 cases = [] 140 with open(filename, 'rb') as f: 141 for line in f: 142 if line[:6] == "TEST: ": 143 cases.append(line[6:].strip()) 144 return cases 145 146 def getCaseList (buildCfg, generator, module): 147 build(buildCfg, generator, [module.binName]) 148 genCaseList(buildCfg, generator, module, "txt") 149 return readCaseList(getCaseListPath(buildCfg, module, "txt")) 150 151 def readPatternList (filename): 152 ptrns = [] 153 with open(filename, 'rb') as f: 154 for line in f: 155 line = line.strip() 156 if len(line) > 0 and line[0] != '#': 157 ptrns.append(line) 158 return ptrns 159 160 def applyPatterns (caseList, patterns, filename, op): 161 matched = set() 162 errors = [] 163 curList = copy(caseList) 164 trivialPtrns = [p for p in patterns if p.find('*') < 0] 165 regularPtrns = [p for p in patterns if p.find('*') >= 0] 166 167 # Apply trivial (just case paths) 168 allCasesSet = set(caseList) 169 for path in trivialPtrns: 170 if path in allCasesSet: 171 if path in matched: 172 errors.append((path, "Same case specified more than once")) 173 matched.add(path) 174 else: 175 errors.append((path, "Test case not found")) 176 177 curList = [c for c in curList if c not in matched] 178 179 for pattern in regularPtrns: 180 matchedThisPtrn = set() 181 182 for case in curList: 183 if fnmatch(case, pattern): 184 matchedThisPtrn.add(case) 185 186 if len(matchedThisPtrn) == 0: 187 errors.append((pattern, "Pattern didn't match any cases")) 188 189 matched = matched | matchedThisPtrn 190 curList = [c for c in curList if c not in matched] 191 192 for pattern, reason in errors: 193 print "ERROR: %s: %s" % (reason, pattern) 194 195 if len(errors) > 0: 196 die("Found %s invalid patterns while processing file %s" % (len(errors), filename)) 197 198 return [c for c in caseList if op(c in matched)] 199 200 def applyInclude (caseList, patterns, filename): 201 return applyPatterns(caseList, patterns, filename, lambda b: b) 202 203 def applyExclude (caseList, patterns, filename): 204 return applyPatterns(caseList, patterns, filename, lambda b: not b) 205 206 def readPatternLists (mustpass): 207 lists = {} 208 for package in mustpass.packages: 209 for cfg in package.configurations: 210 for filter in cfg.filters: 211 if not filter.filename in lists: 212 lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename)) 213 return lists 214 215 def applyFilters (caseList, patternLists, filters): 216 res = copy(caseList) 217 for filter in filters: 218 ptrnList = patternLists[filter.filename] 219 if filter.type == Filter.TYPE_INCLUDE: 220 res = applyInclude(res, ptrnList, filter.filename) 221 else: 222 assert filter.type == Filter.TYPE_EXCLUDE 223 res = applyExclude(res, ptrnList, filter.filename) 224 return res 225 226 def appendToHierarchy (root, casePath): 227 def findChild (node, name): 228 for child in node.children: 229 if child.name == name: 230 return child 231 return None 232 233 curNode = root 234 components = casePath.split('.') 235 236 for component in components[:-1]: 237 nextNode = findChild(curNode, component) 238 if not nextNode: 239 nextNode = TestGroup(component) 240 curNode.children.append(nextNode) 241 curNode = nextNode 242 243 if not findChild(curNode, components[-1]): 244 curNode.children.append(TestCase(components[-1])) 245 246 def buildTestHierachy (caseList): 247 root = TestRoot() 248 for case in caseList: 249 appendToHierarchy(root, case) 250 return root 251 252 def buildTestCaseMap (root): 253 caseMap = {} 254 255 def recursiveBuild (curNode, prefix): 256 curPath = prefix + curNode.name 257 if isinstance(curNode, TestCase): 258 caseMap[curPath] = curNode 259 else: 260 for child in curNode.children: 261 recursiveBuild(child, curPath + '.') 262 263 for child in root.children: 264 recursiveBuild(child, '') 265 266 return caseMap 267 268 def include (filename): 269 return Filter(Filter.TYPE_INCLUDE, filename) 270 271 def exclude (filename): 272 return Filter(Filter.TYPE_EXCLUDE, filename) 273 274 def insertXMLHeaders (mustpass, doc): 275 if mustpass.project.copyright != None: 276 doc.insert(0, ElementTree.Comment(mustpass.project.copyright)) 277 doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING)) 278 279 def prettifyXML (doc): 280 uglyString = ElementTree.tostring(doc, 'utf-8') 281 reparsed = minidom.parseString(uglyString) 282 return reparsed.toprettyxml(indent='\t', encoding='utf-8') 283 284 def genCTSPackageXML (mustpass, package, root): 285 def isLeafGroup (testGroup): 286 numGroups = 0 287 numTests = 0 288 289 for child in testGroup.children: 290 if isinstance(child, TestCase): 291 numTests += 1 292 else: 293 numGroups += 1 294 295 assert numGroups + numTests > 0 296 297 if numGroups > 0 and numTests > 0: 298 die("Mixed groups and cases in %s" % testGroup.name) 299 300 return numGroups == 0 301 302 def makeConfiguration (parentElem, config): 303 attributes = {} 304 305 if config.glconfig != None: 306 attributes['glconfig'] = config.glconfig 307 308 if config.rotation != None: 309 attributes['rotation'] = config.rotation 310 311 if config.surfacetype != None: 312 attributes['surfacetype'] = config.surfacetype 313 314 return ElementTree.SubElement(parentElem, "TestInstance", attributes) 315 316 def makeTestCase (parentElem, testCase): 317 caseElem = ElementTree.SubElement(parentElem, "Test", name=testCase.name) 318 for config in testCase.configurations: 319 makeConfiguration(caseElem, config) 320 return caseElem 321 322 def makeTestGroup (parentElem, testGroup): 323 groupElem = ElementTree.SubElement(parentElem, "TestCase" if isLeafGroup(testGroup) else "TestSuite", name=testGroup.name) 324 for child in testGroup.children: 325 if isinstance(child, TestCase): 326 makeTestCase(groupElem, child) 327 else: 328 makeTestGroup(groupElem, child) 329 return groupElem 330 331 pkgElem = ElementTree.Element("TestPackage", 332 name = package.module.name, 333 appPackageName = getCTSPackageName(package), 334 testType = "deqpTest") 335 336 pkgElem.set("xmlns:deqp", "http://drawelements.com/deqp") 337 insertXMLHeaders(mustpass, pkgElem) 338 339 glesVersion = getModuleGLESVersion(package.module) 340 341 if glesVersion != None: 342 pkgElem.set("deqp:glesVersion", str(glesVersion.encode())) 343 344 for child in root.children: 345 makeTestGroup(pkgElem, child) 346 347 return pkgElem 348 349 def genSpecXML (mustpass): 350 mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version) 351 insertXMLHeaders(mustpass, mustpassElem) 352 353 for package in mustpass.packages: 354 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name) 355 356 for config in package.configurations: 357 configElem = ElementTree.SubElement(packageElem, "Configuration", 358 name = config.name, 359 caseListFile = getCaseListFileName(package, config), 360 commandLine = getCommandLine(config)) 361 362 return mustpassElem 363 364 def addOptionElement (parent, optionName, optionValue): 365 ElementTree.SubElement(parent, "option", name=optionName, value=optionValue) 366 367 def genAndroidTestXml (mustpass): 368 RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner" 369 configElement = ElementTree.Element("configuration") 370 371 for package in mustpass.packages: 372 for config in package.configurations: 373 testElement = ElementTree.SubElement(configElement, "test") 374 testElement.set("class", RUNNER_CLASS) 375 addOptionElement(testElement, "deqp-package", package.module.name) 376 addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config)) 377 # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well. 378 if config.glconfig != None: 379 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig) 380 381 if config.surfacetype != None: 382 addOptionElement(testElement, "deqp-surface-type", config.surfacetype) 383 384 if config.rotation != None: 385 addOptionElement(testElement, "deqp-screen-rotation", config.rotation) 386 387 if config.expectedRuntime != None: 388 addOptionElement(testElement, "runtime-hint", config.expectedRuntime) 389 390 insertXMLHeaders(mustpass, configElement) 391 392 return configElement 393 394 def genMustpass (mustpass, moduleCaseLists): 395 print "Generating mustpass '%s'" % mustpass.version 396 397 patternLists = readPatternLists(mustpass) 398 399 for package in mustpass.packages: 400 allCasesInPkg = moduleCaseLists[package.module] 401 matchingByConfig = {} 402 allMatchingSet = set() 403 404 for config in package.configurations: 405 filtered = applyFilters(allCasesInPkg, patternLists, config.filters) 406 dstFile = getDstCaseListPath(mustpass, package, config) 407 408 print " Writing deqp caselist: " + dstFile 409 writeFile(dstFile, "\n".join(filtered) + "\n") 410 411 matchingByConfig[config] = filtered 412 allMatchingSet = allMatchingSet | set(filtered) 413 414 allMatchingCases = [c for c in allCasesInPkg if c in allMatchingSet] # To preserve ordering 415 root = buildTestHierachy(allMatchingCases) 416 testCaseMap = buildTestCaseMap(root) 417 418 for config in package.configurations: 419 for case in matchingByConfig[config]: 420 testCaseMap[case].configurations.append(config) 421 422 # NOTE: CTS v2 does not need package XML files. Remove when transition is complete. 423 packageXml = genCTSPackageXML(mustpass, package, root) 424 xmlFilename = os.path.join(mustpass.project.path, mustpass.version, getCTSPackageName(package) + ".xml") 425 426 print " Writing CTS caselist: " + xmlFilename 427 writeFile(xmlFilename, prettifyXML(packageXml)) 428 429 specXML = genSpecXML(mustpass) 430 specFilename = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml") 431 432 print " Writing spec: " + specFilename 433 writeFile(specFilename, prettifyXML(specXML)) 434 435 # TODO: Which is the best selector mechanism? 436 if (mustpass.version == "master"): 437 androidTestXML = genAndroidTestXml(mustpass) 438 androidTestFilename = os.path.join(mustpass.project.path, "AndroidTest.xml") 439 440 print " Writing AndroidTest.xml: " + androidTestFilename 441 writeFile(androidTestFilename, prettifyXML(androidTestXML)) 442 443 print "Done!" 444 445 def genMustpassLists (mustpassLists, generator, buildCfg): 446 moduleCaseLists = {} 447 448 # Getting case lists involves invoking build, so we want to cache the results 449 for mustpass in mustpassLists: 450 for package in mustpass.packages: 451 if not package.module in moduleCaseLists: 452 moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module) 453 454 for mustpass in mustpassLists: 455 genMustpass(mustpass, moduleCaseLists) 456
