1 # -*- coding: utf-8 -*- 2 3 #------------------------------------------------------------------------- 4 # drawElements Quality Program utilities 5 # -------------------------------------- 6 # 7 # Copyright 2015 The Android Open Source Project 8 # 9 # Licensed under the Apache License, Version 2.0 (the "License"); 10 # you may not use this file except in compliance with the License. 11 # You may obtain a copy of the License at 12 # 13 # http://www.apache.org/licenses/LICENSE-2.0 14 # 15 # Unless required by applicable law or agreed to in writing, software 16 # distributed under the License is distributed on an "AS IS" BASIS, 17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 # See the License for the specific language governing permissions and 19 # limitations under the License. 20 # 21 #------------------------------------------------------------------------- 22 23 from build.common import * 24 from build.config import ANY_GENERATOR 25 from build.build import build 26 from build_caselists import Module, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET 27 from fnmatch import fnmatch 28 from copy import copy 29 30 import xml.etree.cElementTree as ElementTree 31 import xml.dom.minidom as minidom 32 33 CTS_DATA_DIR = os.path.join(DEQP_DIR, "android", "cts") 34 35 class Configuration: 36 def __init__ (self, name, glconfig, rotation, surfacetype, filters): 37 self.name = name 38 self.glconfig = glconfig 39 self.rotation = rotation 40 self.surfacetype = surfacetype 41 self.filters = filters 42 43 class Package: 44 def __init__ (self, module, configurations): 45 self.module = module 46 self.configurations = configurations 47 48 class Mustpass: 49 def __init__ (self, version, packages): 50 self.version = version 51 self.packages = packages 52 53 class Filter: 54 TYPE_INCLUDE = 0 55 TYPE_EXCLUDE = 1 56 57 def __init__ (self, type, filename): 58 self.type = type 59 self.filename = filename 60 61 class TestRoot: 62 def __init__ (self): 63 self.children = [] 64 65 class TestGroup: 66 def __init__ (self, name): 67 self.name = name 68 self.children = [] 69 70 class TestCase: 71 def __init__ (self, name): 72 self.name = name 73 self.configurations = [] 74 75 class GLESVersion: 76 def __init__(self, major, minor): 77 self.major = major 78 self.minor = minor 79 80 def encode (self): 81 return (self.major << 16) | (self.minor) 82 83 def getModuleGLESVersion (module): 84 versions = { 85 'dEQP-EGL': GLESVersion(2,0), 86 'dEQP-GLES2': GLESVersion(2,0), 87 'dEQP-GLES3': GLESVersion(3,0), 88 'dEQP-GLES31': GLESVersion(3,1) 89 } 90 return versions[module.name] 91 92 def getSrcDir (mustpass): 93 return os.path.join(CTS_DATA_DIR, mustpass.version, "src") 94 95 def getTmpDir (mustpass): 96 return os.path.join(CTS_DATA_DIR, mustpass.version, "tmp") 97 98 def getModuleShorthand (module): 99 assert module.name[:5] == "dEQP-" 100 return module.name[5:].lower() 101 102 def getCaseListFileName (package, configuration): 103 return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name) 104 105 def getDstCaseListPath (mustpass, package, configuration): 106 return os.path.join(CTS_DATA_DIR, mustpass.version, getCaseListFileName(package, configuration)) 107 108 def getCTSPackageName (package): 109 return "com.drawelements.deqp." + getModuleShorthand(package.module) 110 111 def getCommandLine (config): 112 return "--deqp-gl-config-name=%s --deqp-screen-rotation=%s --deqp-surface-type=%s --deqp-watchdog=enable" % (config.glconfig, config.rotation, config.surfacetype) 113 114 def readCaseList (filename): 115 cases = [] 116 with open(filename, 'rb') as f: 117 for line in f: 118 if line[:6] == "TEST: ": 119 cases.append(line[6:].strip()) 120 return cases 121 122 def getCaseList (mustpass, module): 123 generator = ANY_GENERATOR 124 buildCfg = getBuildConfig(DEFAULT_BUILD_DIR, DEFAULT_TARGET, "Debug") 125 126 build(buildCfg, generator, [module.binName]) 127 genCaseList(buildCfg, generator, module, "txt") 128 129 return readCaseList(getCaseListPath(buildCfg, module, "txt")) 130 131 def readPatternList (filename): 132 ptrns = [] 133 with open(filename, 'rb') as f: 134 for line in f: 135 line = line.strip() 136 if len(line) > 0 and line[0] != '#': 137 ptrns.append(line) 138 return ptrns 139 140 def applyPatterns (caseList, patterns, op): 141 matched = set() 142 errors = [] 143 curList = copy(caseList) 144 trivialPtrns = [p for p in patterns if p.find('*') < 0] 145 regularPtrns = [p for p in patterns if p.find('*') >= 0] 146 147 # Apply trivial (just case paths) 148 allCasesSet = set(caseList) 149 for path in trivialPtrns: 150 if path in allCasesSet: 151 if path in matched: 152 errors.append((path, "Same case specified more than once")) 153 matched.add(path) 154 else: 155 errors.append((path, "Test case not found")) 156 157 curList = [c for c in curList if c not in matched] 158 159 for pattern in regularPtrns: 160 matchedThisPtrn = set() 161 162 for case in curList: 163 if fnmatch(case, pattern): 164 matchedThisPtrn.add(case) 165 166 if len(matchedThisPtrn) == 0: 167 errors.append((pattern, "Pattern didn't match any cases")) 168 169 matched = matched | matchedThisPtrn 170 curList = [c for c in curList if c not in matched] 171 172 for pattern, reason in errors: 173 print "ERROR: %s: %s" % (reason, pattern) 174 175 if len(errors) > 0: 176 die("Found %s invalid patterns" % len(errors)) 177 178 return [c for c in caseList if op(c in matched)] 179 180 def applyInclude (caseList, patterns): 181 return applyPatterns(caseList, patterns, lambda b: b) 182 183 def applyExclude (caseList, patterns): 184 return applyPatterns(caseList, patterns, lambda b: not b) 185 186 def readPatternLists (mustpass): 187 lists = {} 188 for package in mustpass.packages: 189 for cfg in package.configurations: 190 for filter in cfg.filters: 191 if not filter.filename in lists: 192 lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename)) 193 return lists 194 195 def applyFilters (caseList, patternLists, filters): 196 res = copy(caseList) 197 for filter in filters: 198 ptrnList = patternLists[filter.filename] 199 if filter.type == Filter.TYPE_INCLUDE: 200 res = applyInclude(res, ptrnList) 201 else: 202 assert filter.type == Filter.TYPE_EXCLUDE 203 res = applyExclude(res, ptrnList) 204 return res 205 206 def appendToHierarchy (root, casePath): 207 def findChild (node, name): 208 for child in node.children: 209 if child.name == name: 210 return child 211 return None 212 213 curNode = root 214 components = casePath.split('.') 215 216 for component in components[:-1]: 217 nextNode = findChild(curNode, component) 218 if not nextNode: 219 nextNode = TestGroup(component) 220 curNode.children.append(nextNode) 221 curNode = nextNode 222 223 if not findChild(curNode, components[-1]): 224 curNode.children.append(TestCase(components[-1])) 225 226 def buildTestHierachy (caseList): 227 root = TestRoot() 228 for case in caseList: 229 appendToHierarchy(root, case) 230 return root 231 232 def buildTestCaseMap (root): 233 caseMap = {} 234 235 def recursiveBuild (curNode, prefix): 236 curPath = prefix + curNode.name 237 if isinstance(curNode, TestCase): 238 caseMap[curPath] = curNode 239 else: 240 for child in curNode.children: 241 recursiveBuild(child, curPath + '.') 242 243 for child in root.children: 244 recursiveBuild(child, '') 245 246 return caseMap 247 248 def include (filename): 249 return Filter(Filter.TYPE_INCLUDE, filename) 250 251 def exclude (filename): 252 return Filter(Filter.TYPE_EXCLUDE, filename) 253 254 def prettifyXML (doc): 255 uglyString = ElementTree.tostring(doc, 'utf-8') 256 reparsed = minidom.parseString(uglyString) 257 return reparsed.toprettyxml(indent='\t', encoding='utf-8') 258 259 def genCTSPackageXML (package, root): 260 def isLeafGroup (testGroup): 261 numGroups = 0 262 numTests = 0 263 264 for child in testGroup.children: 265 if isinstance(child, TestCase): 266 numTests += 1 267 else: 268 numGroups += 1 269 270 assert numGroups + numTests > 0 271 272 if numGroups > 0 and numTests > 0: 273 die("Mixed groups and cases in %s" % testGroup.name) 274 275 return numGroups == 0 276 277 def makeConfiguration (parentElem, configuration): 278 return ElementTree.SubElement(parentElem, "TestInstance", glconfig=configuration.glconfig, rotation=configuration.rotation, surfacetype=configuration.surfacetype) 279 280 def makeTestCase (parentElem, testCase): 281 caseElem = ElementTree.SubElement(parentElem, "Test", name=testCase.name) 282 for config in testCase.configurations: 283 makeConfiguration(caseElem, config) 284 return caseElem 285 286 def makeTestGroup (parentElem, testGroup): 287 groupElem = ElementTree.SubElement(parentElem, "TestCase" if isLeafGroup(testGroup) else "TestSuite", name=testGroup.name) 288 for child in testGroup.children: 289 if isinstance(child, TestCase): 290 makeTestCase(groupElem, child) 291 else: 292 makeTestGroup(groupElem, child) 293 return groupElem 294 295 pkgElem = ElementTree.Element("TestPackage", 296 name = package.module.name, 297 appPackageName = getCTSPackageName(package), 298 testType = "deqpTest") 299 300 pkgElem.set("xmlns:deqp", "http://drawelements.com/deqp") 301 pkgElem.set("deqp:glesVersion", str(getModuleGLESVersion(package.module).encode())) 302 303 for child in root.children: 304 makeTestGroup(pkgElem, child) 305 306 return pkgElem 307 308 def genSpecXML (mustpass): 309 mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version) 310 311 for package in mustpass.packages: 312 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name) 313 314 for config in package.configurations: 315 configElem = ElementTree.SubElement(packageElem, "Configuration", 316 name = config.name, 317 caseListFile = getCaseListFileName(package, config), 318 commandLine = getCommandLine(config)) 319 320 return mustpassElem 321 322 def genMustpass (mustpass, moduleCaseLists): 323 print "Generating mustpass '%s'" % mustpass.version 324 325 patternLists = readPatternLists(mustpass) 326 327 for package in mustpass.packages: 328 allCasesInPkg = moduleCaseLists[package.module] 329 matchingByConfig = {} 330 allMatchingSet = set() 331 332 for config in package.configurations: 333 filtered = applyFilters(allCasesInPkg, patternLists, config.filters) 334 dstFile = getDstCaseListPath(mustpass, package, config) 335 336 print " Writing deqp caselist: " + dstFile 337 writeFile(dstFile, "\n".join(filtered) + "\n") 338 339 matchingByConfig[config] = filtered 340 allMatchingSet = allMatchingSet | set(filtered) 341 342 allMatchingCases = [c for c in allCasesInPkg if c in allMatchingSet] # To preserve ordering 343 root = buildTestHierachy(allMatchingCases) 344 testCaseMap = buildTestCaseMap(root) 345 346 for config in package.configurations: 347 for case in matchingByConfig[config]: 348 testCaseMap[case].configurations.append(config) 349 350 packageXml = genCTSPackageXML(package, root) 351 xmlFilename = os.path.join(CTS_DATA_DIR, mustpass.version, getCTSPackageName(package) + ".xml") 352 353 print " Writing CTS caselist: " + xmlFilename 354 writeFile(xmlFilename, prettifyXML(packageXml)) 355 356 specXML = genSpecXML(mustpass) 357 specFilename = os.path.join(CTS_DATA_DIR, mustpass.version, "mustpass.xml") 358 359 print " Writing spec: " + specFilename 360 writeFile(specFilename, prettifyXML(specXML)) 361 362 print "Done!" 363 364 def genMustpassLists (mustpassLists): 365 moduleCaseLists = {} 366 367 # Getting case lists involves invoking build, so we want to cache the results 368 for mustpass in mustpassLists: 369 for package in mustpass.packages: 370 if not package.module in moduleCaseLists: 371 moduleCaseLists[package.module] = getCaseList(mustpass, package.module) 372 373 for mustpass in mustpassLists: 374 genMustpass(mustpass, moduleCaseLists) 375 376 EGL_MODULE = Module(name = "dEQP-EGL", dirName = "egl", binName = "deqp-egl") 377 GLES2_MODULE = Module(name = "dEQP-GLES2", dirName = "gles2", binName = "deqp-gles2") 378 GLES3_MODULE = Module(name = "dEQP-GLES3", dirName = "gles3", binName = "deqp-gles3") 379 GLES31_MODULE = Module(name = "dEQP-GLES31", dirName = "gles31", binName = "deqp-gles31") 380 381 LMP_GLES3_PKG = Package(module = GLES3_MODULE, configurations = [ 382 Configuration(name = "master", 383 glconfig = "rgba8888d24s8ms0", 384 rotation = "unspecified", 385 surfacetype = "window", 386 filters = [include("es30-lmp.txt")]), 387 ]) 388 LMP_GLES31_PKG = Package(module = GLES31_MODULE, configurations = [ 389 Configuration(name = "master", 390 glconfig = "rgba8888d24s8ms0", 391 rotation = "unspecified", 392 surfacetype = "window", 393 filters = [include("es31-lmp.txt")]), 394 ]) 395 396 LMP_MR1_GLES3_PKG = Package(module = GLES3_MODULE, configurations = [ 397 Configuration(name = "master", 398 glconfig = "rgba8888d24s8ms0", 399 rotation = "unspecified", 400 surfacetype = "window", 401 filters = [include("es30-lmp-mr1.txt")]), 402 ]) 403 LMP_MR1_GLES31_PKG = Package(module = GLES31_MODULE, configurations = [ 404 Configuration(name = "master", 405 glconfig = "rgba8888d24s8ms0", 406 rotation = "unspecified", 407 surfacetype = "window", 408 filters = [include("es31-lmp-mr1.txt")]), 409 ]) 410 411 MASTER_EGL_COMMON_FILTERS = [include("egl-master.txt"), exclude("egl-failures.txt")] 412 MASTER_EGL_PKG = Package(module = EGL_MODULE, configurations = [ 413 # Master 414 Configuration(name = "master", 415 glconfig = "rgba8888d24s8ms0", 416 rotation = "unspecified", 417 surfacetype = "window", 418 filters = MASTER_EGL_COMMON_FILTERS), 419 ]) 420 421 MASTER_GLES2_COMMON_FILTERS = [ 422 include("gles2-master.txt"), 423 exclude("gles2-test-issues.txt"), 424 exclude("gles2-failures.txt") 425 ] 426 MASTER_GLES2_PKG = Package(module = GLES2_MODULE, configurations = [ 427 # Master 428 Configuration(name = "master", 429 glconfig = "rgba8888d24s8ms0", 430 rotation = "unspecified", 431 surfacetype = "window", 432 filters = MASTER_GLES2_COMMON_FILTERS), 433 ]) 434 435 MASTER_GLES3_COMMON_FILTERS = [ 436 include("gles3-master.txt"), 437 exclude("gles3-hw-issues.txt"), 438 exclude("gles3-driver-issues.txt"), 439 exclude("gles3-test-issues.txt"), 440 exclude("gles3-spec-issues.txt") 441 ] 442 MASTER_GLES3_PKG = Package(module = GLES3_MODULE, configurations = [ 443 # Master 444 Configuration(name = "master", 445 glconfig = "rgba8888d24s8ms0", 446 rotation = "unspecified", 447 surfacetype = "window", 448 filters = MASTER_GLES3_COMMON_FILTERS), 449 # Rotations 450 Configuration(name = "rotate-portrait", 451 glconfig = "rgba8888d24s8ms0", 452 rotation = "0", 453 surfacetype = "window", 454 filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]), 455 Configuration(name = "rotate-landscape", 456 glconfig = "rgba8888d24s8ms0", 457 rotation = "90", 458 surfacetype = "window", 459 filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]), 460 Configuration(name = "rotate-reverse-portrait", 461 glconfig = "rgba8888d24s8ms0", 462 rotation = "180", 463 surfacetype = "window", 464 filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]), 465 Configuration(name = "rotate-reverse-landscape", 466 glconfig = "rgba8888d24s8ms0", 467 rotation = "270", 468 surfacetype = "window", 469 filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]), 470 471 # MSAA 472 Configuration(name = "multisample", 473 glconfig = "rgba8888d24s8ms4", 474 rotation = "unspecified", 475 surfacetype = "window", 476 filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-multisample.txt"), 477 exclude("gles3-multisample-issues.txt")]), 478 479 # Pixel format 480 Configuration(name = "565-no-depth-no-stencil", 481 glconfig = "rgb565d0s0ms0", 482 rotation = "unspecified", 483 surfacetype = "window", 484 filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-pixelformat.txt"), 485 exclude("gles3-pixelformat-issues.txt")]), 486 ]) 487 488 MASTER_GLES31_COMMON_FILTERS = [ 489 include("gles31-master.txt"), 490 exclude("gles31-hw-issues.txt"), 491 exclude("gles31-driver-issues.txt"), 492 exclude("gles31-test-issues.txt"), 493 exclude("gles31-spec-issues.txt"), 494 ] 495 MASTER_GLES31_PKG = Package(module = GLES31_MODULE, configurations = [ 496 # Master 497 Configuration(name = "master", 498 glconfig = "rgba8888d24s8ms0", 499 rotation = "unspecified", 500 surfacetype = "window", 501 filters = MASTER_GLES31_COMMON_FILTERS), 502 503 # Rotations 504 Configuration(name = "rotate-portrait", 505 glconfig = "rgba8888d24s8ms0", 506 rotation = "0", 507 surfacetype = "window", 508 filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]), 509 Configuration(name = "rotate-landscape", 510 glconfig = "rgba8888d24s8ms0", 511 rotation = "90", 512 surfacetype = "window", 513 filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]), 514 Configuration(name = "rotate-reverse-portrait", 515 glconfig = "rgba8888d24s8ms0", 516 rotation = "180", 517 surfacetype = "window", 518 filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]), 519 Configuration(name = "rotate-reverse-landscape", 520 glconfig = "rgba8888d24s8ms0", 521 rotation = "270", 522 surfacetype = "window", 523 filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]), 524 525 # MSAA 526 Configuration(name = "multisample", 527 glconfig = "rgba8888d24s8ms4", 528 rotation = "unspecified", 529 surfacetype = "window", 530 filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-multisample.txt")]), 531 532 # Pixel format 533 Configuration(name = "565-no-depth-no-stencil", 534 glconfig = "rgb565d0s0ms0", 535 rotation = "unspecified", 536 surfacetype = "window", 537 filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-pixelformat.txt")]), 538 ]) 539 540 MUSTPASS_LISTS = [ 541 Mustpass(version = "lmp", packages = [LMP_GLES3_PKG, LMP_GLES31_PKG]), 542 Mustpass(version = "lmp-mr1", packages = [LMP_MR1_GLES3_PKG, LMP_MR1_GLES31_PKG]), 543 Mustpass(version = "master", packages = [MASTER_EGL_PKG, MASTER_GLES2_PKG, MASTER_GLES3_PKG, MASTER_GLES31_PKG]) 544 ] 545 546 if __name__ == "__main__": 547 genMustpassLists(MUSTPASS_LISTS) 548
