1 # -*- coding: utf-8 -*- 2 3 #------------------------------------------------------------------------- 4 # Khronos OpenGL CTS 5 # ------------------ 6 # 7 # Copyright (c) 2016 The Khronos Group Inc. 8 # 9 # Licensed under the Apache License, Version 2.0 (the "License"); 10 # you may not use this file except in compliance with the License. 11 # You may obtain a copy of the License at 12 # 13 # http://www.apache.org/licenses/LICENSE-2.0 14 # 15 # Unless required by applicable law or agreed to in writing, software 16 # distributed under the License is distributed on an "AS IS" BASIS, 17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 # See the License for the specific language governing permissions and 19 # limitations under the License. 20 # 21 #------------------------------------------------------------------------- 22 23 import os 24 import sys 25 import xml.dom.minidom 26 import re 27 import subprocess 28 29 ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) 30 sys.path.append(os.path.join(ROOT_DIR, "scripts", "verify")) 31 sys.path.append(os.path.join(ROOT_DIR, "scripts", "build")) 32 sys.path.append(os.path.join(ROOT_DIR, "scripts", "log")) 33 34 from package import getPackageDescription 35 from verify import * 36 from message import * 37 from common import * 38 from log_parser import * 39 from summary import * 40 41 def getConfigCaseName (type): 42 configs = { "es32" : ["CTS-Configs.es32", "CTS-Configs.es31", "CTS-Configs.es3", "CTS-Configs.es2"], 43 "es31" : ["CTS-Configs.es31", "CTS-Configs.es3", "CTS-Configs.es2"], 44 "es3" : ["CTS-Configs.es3", "CTS-Configs.es2"], 45 "es2" : ["CTS-Configs.es2"]} 46 return configs[type] 47 48 def retrieveReportedConfigs(caseName, log): 49 doc = xml.dom.minidom.parseString(log) 50 sectionItems = doc.getElementsByTagName('Section') 51 sectionName = None 52 53 configs = [] 54 for sectionItem in sectionItems: 55 sectionName = sectionItem.getAttributeNode('Name').nodeValue 56 if sectionName == "Configs": 57 assert len(configs) == 0 58 textItems = sectionItem.getElementsByTagName('Text') 59 for textItem in textItems: 60 configs.append(getNodeText(textItem)) 61 res = {caseName : configs} 62 return res 63 64 def compareConfigs(filename, baseConfigs, cmpConfigs): 65 messages = [] 66 assert len(list(baseConfigs.keys())) == 1 67 assert len(list(cmpConfigs.keys())) == 1 68 baseKey = list(baseConfigs.keys())[0] 69 cmpKey = list(cmpConfigs.keys())[0] 70 71 if cmp(baseConfigs[baseKey], cmpConfigs[cmpKey]) != 0: 72 messages.append(error(filename, "Confomant configs reported for %s and %s do not match" % (baseKey,cmpKey))) 73 74 return messages 75 76 def verifyConfigFile (filename, type): 77 messages = [] 78 caseNames = getConfigCaseName(type) 79 80 parser = BatchResultParser() 81 results = parser.parseFile(filename) 82 baseConfigs = None 83 84 for caseName in caseNames: 85 caseResult = None 86 print "Verifying %s in %s" % (caseName, filename) 87 for result in results: 88 if result.name == caseName: 89 caseResult = result 90 break; 91 if caseResult == None: 92 messages.append(error(filename, "Missing %s" % caseName)) 93 else: 94 configs = retrieveReportedConfigs(caseName, result.log) 95 if baseConfigs == None: 96 baseConfigs = configs 97 else: 98 messages += compareConfigs(filename, baseConfigs, configs) 99 if not caseResult.statusCode in ALLOWED_STATUS_CODES: 100 messages.append(error(filename, "%s failed" % caseResult)) 101 102 return messages 103 104 def verifyMustpassCases(package, mustpassCases, type): 105 messages = [] 106 apiToTest = { "es32" : ["gles32", "gles31", "gles3", "gles2", "egl"], 107 "es31" : ["gles31", "gles3", "gles2", "egl"], 108 "es3" : ["gles3", "gles2", "egl"], 109 "es2" : ["gles2", "egl"]} 110 111 for mustpass in mustpassCases: 112 mustpassXML = os.path.join(mustpass, "mustpass.xml") 113 doc = xml.dom.minidom.parse(mustpassXML) 114 testConfigs = doc.getElementsByTagName("Configuration") 115 # check that all configs that must be tested are present 116 for testConfig in testConfigs: 117 caseListFile = testConfig.getAttributeNode("caseListFile").nodeValue 118 # identify APIs that must be tested for the given type 119 apis = apiToTest[type] 120 # identify API tested by the current config 121 configAPI = caseListFile.split('-')[0] 122 if configAPI in apis: 123 # the API in this config is expected to be tested 124 mustTest = True 125 else: 126 mustTest = False 127 pattern = "config-" + os.path.splitext(caseListFile)[0] + "-cfg-[0-9]*"+"-run-[0-9]*" 128 cmdLine = testConfig.getAttributeNode("commandLine").nodeValue 129 cfgItems = {'height':None, 'width':None, 'seed':None, 'rotation':None} 130 for arg in cmdLine.split(): 131 val = arg.split('=')[1] 132 if "deqp-surface-height" in arg: 133 cfgItems['height'] = val 134 elif "deqp-surface-width" in arg: 135 cfgItems['width'] = val 136 elif "deqp-base-seed" in arg: 137 cfgItems['seed'] = val 138 elif "deqp-screen-rotation" in arg: 139 cfgItems['rotation'] = val 140 pattern += "-width-" + cfgItems['width'] + "-height-" + cfgItems['height'] 141 if cfgItems['seed'] != None: 142 pattern += "-seed-" + cfgItems['seed'] 143 pattern += ".qpa" 144 p = re.compile(pattern) 145 matches = [m for l in mustpassCases[mustpass] for m in (p.match(l),) if m] 146 if len(matches) == 0 and mustTest == True: 147 conformOs = testConfig.getAttributeNode("os").nodeValue 148 txt = "Configuration %s %s was not executed" % (caseListFile, cmdLine) 149 if conformOs == "any" or (package.conformOs != None and conformOs in package.conformOs.lower()): 150 msg = error(mustpassXML, txt) 151 else: 152 msg = warning(mustpassXML, txt) 153 messages.append(msg) 154 elif len(matches) != 0 and mustTest == False: 155 messages.append(error(mustpassXML, "Configuration %s %s was not expected to be tested but present in cts-run-summary.xml" % (caseListFile, cmdLine))) 156 157 return messages 158 159 def verifyTestLogs (package): 160 messages = [] 161 162 try: 163 execute(['git', 'checkout', '--quiet', package.conformVersion]) 164 except Exception, e: 165 print str(e) 166 print "Failed to checkout release tag %s." % package.conformVersion 167 return messages 168 169 messages = [] 170 summary = parseRunSummary(os.path.join(package.basePath, package.summary)) 171 mustpassDirs = [] 172 173 # Check Conformant attribute 174 if not summary.isConformant: 175 messages.append(error(package.summary, "Runner reported conformance failure (Conformant=\"False\" in <Summary>)")) 176 177 # Verify config list 178 messages += verifyConfigFile(os.path.join(package.basePath, summary.configLogFilename), summary.type) 179 180 mustpassCases = {} 181 # Verify that all run files passed 182 for runLog in summary.runLogAndCaselist: 183 sys.stdout.write("Verifying %s -" % runLog) 184 sys.stdout.flush() 185 186 mustpassFile = os.path.join(ROOT_DIR, "external", "openglcts", summary.runLogAndCaselist[runLog]) 187 key = os.path.dirname(mustpassFile) 188 if key in mustpassCases: 189 mpCase = mustpassCases[key] 190 else: 191 mpCase = [] 192 mpCase.append(runLog) 193 mustpassCases[os.path.dirname(mustpassFile)] = mpCase 194 mustpass = readMustpass(mustpassFile) 195 messages_log = verifyTestLog(os.path.join(package.basePath, runLog), mustpass) 196 197 errors = [m for m in messages_log if m.type == ValidationMessage.TYPE_ERROR] 198 warnings = [m for m in messages_log if m.type == ValidationMessage.TYPE_WARNING] 199 if len(errors) > 0: 200 sys.stdout.write(" finished with ERRRORS") 201 if len(warnings) > 0: 202 sys.stdout.write(" finished with WARNINGS") 203 if len(errors) == 0 and len(warnings) == 0: 204 sys.stdout.write(" OK") 205 sys.stdout.write("\n") 206 sys.stdout.flush() 207 208 messages += messages_log 209 210 messages += verifyMustpassCases(package, mustpassCases, summary.type) 211 212 return messages 213 214 def verifyGitStatusFiles (package): 215 messages = [] 216 217 if len(package.gitStatus) != 2: 218 messages.append(error(package.basePath, "Exactly two git status files must be present, found %s" % len(package.gitStatus))) 219 220 messages += verifyGitStatus(package) 221 222 return messages 223 224 def isGitLogFileEmpty (package, modulePath, gitLog): 225 logPath = os.path.join(package.basePath, gitLog) 226 log = readFile(logPath) 227 228 args = ['git', 'log', '-1', package.conformVersion] 229 process = subprocess.Popen(args, cwd=modulePath, stdout=subprocess.PIPE) 230 output = process.communicate()[0] 231 if process.returncode != 0: 232 raise Exception("Failed to execute '%s', got %d" % (str(args), process.returncode)) 233 234 return log == output 235 236 def verifyGitLogFile (package): 237 messages = [] 238 239 if len(package.gitLog) > 0: 240 for log, path in package.gitLog: 241 try: 242 isEmpty = isGitLogFileEmpty(package, path, log) 243 except Exception, e: 244 print str(e) 245 isEmpty = False 246 247 if not isEmpty: 248 messages.append(warning(os.path.join(package.basePath, log), "Log is not empty")) 249 else: 250 messages.append(error(package.basePath, "Missing git log files")) 251 252 return messages 253 254 def verifyPatchFiles (package): 255 messages = [] 256 hasPatches = len(package.patches) 257 logEmpty = True 258 for log, path in package.gitLog: 259 logEmpty &= isGitLogFileEmpty(package, path, log) 260 261 if hasPatches and logEmpty: 262 messages.append(error(package.basePath, "Package includes patches but log is empty")) 263 elif not hasPatches and not logEmpty: 264 messages.append(error(package.basePath, "Test log is not empty but package doesn't contain patches")) 265 266 return messages 267 268 def verifyGitLogFiles (package): 269 messages = [] 270 271 if len(package.gitLog) != 2: 272 messages.append(error(package.basePath, "Exactly two git log file must be present, found %s" % len(package.gitLog))) 273 274 for i, gitLog in enumerate(package.gitLog): 275 if "kc-cts" in gitLog[0]: 276 package.gitLog[i] = gitLog[:1] + ("external/kc-cts/src",) + gitLog[2:] 277 278 messages += verifyGitLogFile(package) 279 280 return messages 281 282 def verifyPackage (package): 283 messages = [] 284 285 messages += verifyStatement(package) 286 messages += verifyGitStatusFiles(package) 287 messages += verifyGitLogFiles(package) 288 messages += verifyPatchFiles(package) 289 290 for item in package.otherItems: 291 messages.append(warning(os.path.join(package.basePath, item), "Unknown file")) 292 293 return messages 294 295 def verifyESSubmission(argv): 296 if len(argv) != 2: 297 print "%s: [extracted submission package directory]" % sys.argv[0] 298 sys.exit(-1) 299 try: 300 execute(['git', 'ls-remote', 'origin', '--quiet']) 301 except Exception, e: 302 print str(e) 303 print "This script must be executed inside VK-GL-CTS directory." 304 sys.exit(-1) 305 306 packagePath = os.path.normpath(sys.argv[1]) 307 package = getPackageDescription(packagePath) 308 messages = verifyPackage(package) 309 messages += verifyTestLogs(package) 310 311 errors = [m for m in messages if m.type == ValidationMessage.TYPE_ERROR] 312 warnings = [m for m in messages if m.type == ValidationMessage.TYPE_WARNING] 313 314 for message in messages: 315 print str(message) 316 317 print "" 318 319 if len(errors) > 0: 320 print "Found %d validation errors and %d warnings!" % (len(errors), len(warnings)) 321 sys.exit(-2) 322 elif len(warnings) > 0: 323 print "Found %d warnings, manual review required" % len(warnings) 324 sys.exit(-1) 325 else: 326 print "All validation checks passed" 327 328 if __name__ == "__main__": 329 verifyESSubmission(sys.argv) 330
