1 # -*- coding: utf-8 -*- 2 3 #------------------------------------------------------------------------- 4 # drawElements Quality Program utilities 5 # -------------------------------------- 6 # 7 # Copyright 2015 The Android Open Source Project 8 # 9 # Licensed under the Apache License, Version 2.0 (the "License"); 10 # you may not use this file except in compliance with the License. 11 # You may obtain a copy of the License at 12 # 13 # http://www.apache.org/licenses/LICENSE-2.0 14 # 15 # Unless required by applicable law or agreed to in writing, software 16 # distributed under the License is distributed on an "AS IS" BASIS, 17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 # See the License for the specific language governing permissions and 19 # limitations under the License. 20 # 21 #------------------------------------------------------------------------- 22 23 from build.common import * 24 from build.config import * 25 from build.build import * 26 27 import os 28 import sys 29 import string 30 import socket 31 import fnmatch 32 from datetime import datetime 33 34 BASE_NIGHTLY_DIR = os.path.normpath(os.path.join(DEQP_DIR, "..", "deqp-nightly")) 35 BASE_BUILD_DIR = os.path.join(BASE_NIGHTLY_DIR, "build") 36 BASE_LOGS_DIR = os.path.join(BASE_NIGHTLY_DIR, "logs") 37 BASE_REFS_DIR = os.path.join(BASE_NIGHTLY_DIR, "refs") 38 39 EXECUTOR_PATH = "executor/executor" 40 LOG_TO_CSV_PATH = "executor/testlog-to-csv" 41 EXECSERVER_PATH = "execserver/execserver" 42 43 CASELIST_PATH = os.path.join(DEQP_DIR, "Candy", "Data") 44 45 COMPARE_NUM_RESULTS = 4 46 COMPARE_REPORT_NAME = "nightly-report.html" 47 48 COMPARE_REPORT_TMPL = ''' 49 <html> 50 <head> 51 <title>${TITLE}</title> 52 <style type="text/css"> 53 <!-- 54 body { font: serif; font-size: 1em; } 55 table { border-spacing: 0; border-collapse: collapse; } 56 td { border-width: 1px; border-style: solid; border-color: #808080; } 57 .Header { font-weight: bold; font-size: 1em; border-style: none; } 58 .CasePath { } 59 .Pass { background: #80ff80; } 60 .Fail { background: #ff4040; } 61 .QualityWarning { background: #ffff00; } 62 .CompabilityWarning { background: #ffff00; } 63 .Pending { background: #808080; } 64 .Running { background: #d3d3d3; } 65 .NotSupported { background: #ff69b4; } 66 .ResourceError { background: #ff4040; } 67 .InternalError { background: #ff1493; } 68 .Canceled { background: #808080; } 69 .Crash { background: #ffa500; } 70 .Timeout { background: #ffa500; } 71 .Disabled { background: #808080; } 72 .Missing { background: #808080; } 73 .Ignored { opacity: 0.5; } 74 --> 75 </style> 76 </head> 77 <body> 78 <h1>${TITLE}</h1> 79 <table> 80 ${RESULTS} 81 </table> 82 </body> 83 </html> 84 ''' 85 86 class NightlyRunConfig: 87 def __init__(self, name, buildConfig, generator, binaryName, testset, args = [], exclude = [], ignore = []): 88 self.name = name 89 self.buildConfig = buildConfig 90 self.generator = generator 91 self.binaryName = binaryName 92 self.testset = testset 93 self.args = args 94 self.exclude = exclude 95 self.ignore = ignore 96 97 def getBinaryPath(self, basePath): 98 return os.path.join(self.buildConfig.getBuildDir(), self.generator.getBinaryPath(self.buildConfig.getBuildType(), basePath)) 99 100 class NightlyBuildConfig(BuildConfig): 101 def __init__(self, name, buildType, args): 102 BuildConfig.__init__(self, os.path.join(BASE_BUILD_DIR, name), buildType, args) 103 104 class TestCaseResult: 105 def __init__ (self, name, statusCode): 106 self.name = name 107 self.statusCode = statusCode 108 109 class MultiResult: 110 def __init__ (self, name, statusCodes): 111 self.name = name 112 self.statusCodes = statusCodes 113 114 class BatchResult: 115 def __init__ (self, name): 116 self.name = name 117 self.results = [] 118 119 def parseResultCsv (data): 120 lines = data.splitlines()[1:] 121 results = [] 122 123 for line in lines: 124 items = line.split(",") 125 results.append(TestCaseResult(items[0], items[1])) 126 127 return results 128 129 def readTestCaseResultsFromCSV (filename): 130 return parseResultCsv(readFile(filename)) 131 132 def readBatchResultFromCSV (filename, batchResultName = None): 133 batchResult = BatchResult(batchResultName if batchResultName != None else os.path.basename(filename)) 134 batchResult.results = readTestCaseResultsFromCSV(filename) 135 return batchResult 136 137 def getResultTimestamp (): 138 return datetime.now().strftime("%Y-%m-%d-%H-%M") 139 140 def getCompareFilenames (logsDir): 141 files = [] 142 for file in os.listdir(logsDir): 143 fullPath = os.path.join(logsDir, file) 144 if os.path.isfile(fullPath) and fnmatch.fnmatch(file, "*.csv"): 145 files.append(fullPath) 146 files.sort() 147 148 return files[-COMPARE_NUM_RESULTS:] 149 150 def parseAsCSV (logPath, config): 151 args = [config.getBinaryPath(LOG_TO_CSV_PATH), "--mode=all", "--format=csv", logPath] 152 proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 153 out, err = proc.communicate() 154 return out 155 156 def computeUnifiedTestCaseList (batchResults): 157 caseList = [] 158 caseSet = set() 159 160 for batchResult in batchResults: 161 for result in batchResult.results: 162 if not result.name in caseSet: 163 caseList.append(result.name) 164 caseSet.add(result.name) 165 166 return caseList 167 168 def computeUnifiedResults (batchResults): 169 170 def genResultMap (batchResult): 171 resMap = {} 172 for result in batchResult.results: 173 resMap[result.name] = result 174 return resMap 175 176 resultMap = [genResultMap(r) for r in batchResults] 177 caseList = computeUnifiedTestCaseList(batchResults) 178 results = [] 179 180 for caseName in caseList: 181 statusCodes = [] 182 183 for i in range(0, len(batchResults)): 184 result = resultMap[i][caseName] if caseName in resultMap[i] else None 185 statusCode = result.statusCode if result != None else 'Missing' 186 statusCodes.append(statusCode) 187 188 results.append(MultiResult(caseName, statusCodes)) 189 190 return results 191 192 def allStatusCodesEqual (result): 193 firstCode = result.statusCodes[0] 194 for i in range(1, len(result.statusCodes)): 195 if result.statusCodes[i] != firstCode: 196 return False 197 return True 198 199 def computeDiffResults (unifiedResults): 200 diff = [] 201 for result in unifiedResults: 202 if not allStatusCodesEqual(result): 203 diff.append(result) 204 return diff 205 206 def genCompareReport (batchResults, title, ignoreCases): 207 class TableRow: 208 def __init__ (self, testCaseName, innerHTML): 209 self.testCaseName = testCaseName 210 self.innerHTML = innerHTML 211 212 unifiedResults = computeUnifiedResults(batchResults) 213 diffResults = computeDiffResults(unifiedResults) 214 rows = [] 215 216 # header 217 headerCol = '<td class="Header">Test case</td>\n' 218 for batchResult in batchResults: 219 headerCol += '<td class="Header">%s</td>\n' % batchResult.name 220 rows.append(TableRow(None, headerCol)) 221 222 # results 223 for result in diffResults: 224 col = '<td class="CasePath">%s</td>\n' % result.name 225 for statusCode in result.statusCodes: 226 col += '<td class="%s">%s</td>\n' % (statusCode, statusCode) 227 228 rows.append(TableRow(result.name, col)) 229 230 tableStr = "" 231 for row in rows: 232 if row.testCaseName is not None and matchesAnyPattern(row.testCaseName, ignoreCases): 233 tableStr += '<tr class="Ignored">\n%s</tr>\n' % row.innerHTML 234 else: 235 tableStr += '<tr>\n%s</tr>\n' % row.innerHTML 236 237 html = COMPARE_REPORT_TMPL 238 html = html.replace("${TITLE}", title) 239 html = html.replace("${RESULTS}", tableStr) 240 241 return html 242 243 def matchesAnyPattern (name, patterns): 244 for pattern in patterns: 245 if fnmatch.fnmatch(name, pattern): 246 return True 247 return False 248 249 def statusCodesMatch (refResult, resResult): 250 return refResult == 'Missing' or resResult == 'Missing' or refResult == resResult 251 252 def compareBatchResults (referenceBatch, resultBatch, ignoreCases): 253 unifiedResults = computeUnifiedResults([referenceBatch, resultBatch]) 254 failedCases = [] 255 256 for result in unifiedResults: 257 if not matchesAnyPattern(result.name, ignoreCases): 258 refResult = result.statusCodes[0] 259 resResult = result.statusCodes[1] 260 261 if not statusCodesMatch(refResult, resResult): 262 failedCases.append(result) 263 264 return failedCases 265 266 def getUnusedPort (): 267 # \note Not 100%-proof method as other apps may grab this port before we launch execserver 268 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 269 s.bind(('localhost', 0)) 270 addr, port = s.getsockname() 271 s.close() 272 return port 273 274 def runNightly (config): 275 build(config.buildConfig, config.generator) 276 277 # Run parameters 278 timestamp = getResultTimestamp() 279 logDir = os.path.join(BASE_LOGS_DIR, config.name) 280 testLogPath = os.path.join(logDir, timestamp + ".qpa") 281 infoLogPath = os.path.join(logDir, timestamp + ".txt") 282 csvLogPath = os.path.join(logDir, timestamp + ".csv") 283 compareLogPath = os.path.join(BASE_REFS_DIR, config.name + ".csv") 284 port = getUnusedPort() 285 286 if not os.path.exists(logDir): 287 os.makedirs(logDir) 288 289 if os.path.exists(testLogPath) or os.path.exists(infoLogPath): 290 raise Exception("Result '%s' already exists", timestamp) 291 292 # Paths, etc. 293 binaryName = config.generator.getBinaryPath(config.buildConfig.getBuildType(), os.path.basename(config.binaryName)) 294 workingDir = os.path.join(config.buildConfig.getBuildDir(), os.path.dirname(config.binaryName)) 295 296 execArgs = [ 297 config.getBinaryPath(EXECUTOR_PATH), 298 '--start-server=%s' % config.getBinaryPath(EXECSERVER_PATH), 299 '--port=%d' % port, 300 '--binaryname=%s' % binaryName, 301 '--cmdline=%s' % string.join([shellquote(arg) for arg in config.args], " "), 302 '--workdir=%s' % workingDir, 303 '--caselistdir=%s' % CASELIST_PATH, 304 '--testset=%s' % string.join(config.testset, ","), 305 '--out=%s' % testLogPath, 306 '--info=%s' % infoLogPath, 307 '--summary=no' 308 ] 309 310 if len(config.exclude) > 0: 311 execArgs += ['--exclude=%s' % string.join(config.exclude, ",")] 312 313 execute(execArgs) 314 315 # Translate to CSV for comparison purposes 316 lastResultCsv = parseAsCSV(testLogPath, config) 317 writeFile(csvLogPath, lastResultCsv) 318 319 if os.path.exists(compareLogPath): 320 refBatchResult = readBatchResultFromCSV(compareLogPath, "reference") 321 else: 322 refBatchResult = None 323 324 # Generate comparison report 325 compareFilenames = getCompareFilenames(logDir) 326 batchResults = [readBatchResultFromCSV(filename) for filename in compareFilenames] 327 328 if refBatchResult != None: 329 batchResults = [refBatchResult] + batchResults 330 331 writeFile(COMPARE_REPORT_NAME, genCompareReport(batchResults, config.name, config.ignore)) 332 print "Comparison report written to %s" % COMPARE_REPORT_NAME 333 334 # Compare to reference 335 if refBatchResult != None: 336 curBatchResult = BatchResult("current") 337 curBatchResult.results = parseResultCsv(lastResultCsv) 338 failedCases = compareBatchResults(refBatchResult, curBatchResult, config.ignore) 339 340 print "" 341 for result in failedCases: 342 print "MISMATCH: %s: expected %s, got %s" % (result.name, result.statusCodes[0], result.statusCodes[1]) 343 344 print "" 345 print "%d / %d cases passed, run %s" % (len(curBatchResult.results)-len(failedCases), len(curBatchResult.results), "FAILED" if len(failedCases) > 0 else "passed") 346 347 if len(failedCases) > 0: 348 return False 349 350 return True 351 352 # Configurations 353 354 DEFAULT_WIN32_GENERATOR = ANY_VS_X32_GENERATOR 355 DEFAULT_WIN64_GENERATOR = ANY_VS_X64_GENERATOR 356 357 WGL_X64_RELEASE_BUILD_CFG = NightlyBuildConfig("wgl_x64_release", "Release", ['-DDEQP_TARGET=win32_wgl']) 358 ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG = NightlyBuildConfig("arm_gles3_emu_release", "Release", ['-DDEQP_TARGET=arm_gles3_emu']) 359 360 BASE_ARGS = ['--deqp-visibility=hidden', '--deqp-watchdog=enable', '--deqp-crashhandler=enable'] 361 362 CONFIGS = [ 363 NightlyRunConfig( 364 name = "wgl_x64_release_gles2", 365 buildConfig = WGL_X64_RELEASE_BUILD_CFG, 366 generator = DEFAULT_WIN64_GENERATOR, 367 binaryName = "modules/gles2/deqp-gles2", 368 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 369 testset = ["dEQP-GLES2.info.*", "dEQP-GLES2.functional.*", "dEQP-GLES2.usecases.*"], 370 exclude = [ 371 "dEQP-GLES2.functional.shaders.loops.*while*unconditional_continue*", 372 "dEQP-GLES2.functional.shaders.loops.*while*only_continue*", 373 "dEQP-GLES2.functional.shaders.loops.*while*double_continue*", 374 ], 375 ignore = [] 376 ), 377 NightlyRunConfig( 378 name = "wgl_x64_release_gles3", 379 buildConfig = WGL_X64_RELEASE_BUILD_CFG, 380 generator = DEFAULT_WIN64_GENERATOR, 381 binaryName = "modules/gles3/deqp-gles3", 382 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 383 testset = ["dEQP-GLES3.info.*", "dEQP-GLES3.functional.*", "dEQP-GLES3.usecases.*"], 384 exclude = [ 385 "dEQP-GLES3.functional.shaders.loops.*while*unconditional_continue*", 386 "dEQP-GLES3.functional.shaders.loops.*while*only_continue*", 387 "dEQP-GLES3.functional.shaders.loops.*while*double_continue*", 388 ], 389 ignore = [ 390 "dEQP-GLES3.functional.transform_feedback.*", 391 "dEQP-GLES3.functional.occlusion_query.*", 392 "dEQP-GLES3.functional.lifetime.*", 393 "dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_ops", 394 ] 395 ), 396 NightlyRunConfig( 397 name = "wgl_x64_release_gles31", 398 buildConfig = WGL_X64_RELEASE_BUILD_CFG, 399 generator = DEFAULT_WIN64_GENERATOR, 400 binaryName = "modules/gles31/deqp-gles31", 401 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 402 testset = ["dEQP-GLES31.*"], 403 exclude = [], 404 ignore = [ 405 "dEQP-GLES31.functional.draw_indirect.negative.command_bad_alignment_3", 406 "dEQP-GLES31.functional.draw_indirect.negative.command_offset_not_in_buffer", 407 "dEQP-GLES31.functional.vertex_attribute_binding.negative.bind_vertex_buffer_negative_offset", 408 "dEQP-GLES31.functional.ssbo.layout.single_basic_type.packed.mediump_uint", 409 "dEQP-GLES31.functional.blend_equation_advanced.basic.*", 410 "dEQP-GLES31.functional.blend_equation_advanced.srgb.*", 411 "dEQP-GLES31.functional.blend_equation_advanced.barrier.*", 412 "dEQP-GLES31.functional.uniform_location.*", 413 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_framebuffer_attachment_parameteriv", 414 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_renderbuffer_parameteriv", 415 "dEQP-GLES31.functional.debug.error_filters.case_0", 416 "dEQP-GLES31.functional.debug.error_filters.case_2", 417 ] 418 ), 419 NightlyRunConfig( 420 name = "wgl_x64_release_gl3", 421 buildConfig = WGL_X64_RELEASE_BUILD_CFG, 422 generator = DEFAULT_WIN64_GENERATOR, 423 binaryName = "modules/gl3/deqp-gl3", 424 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 425 testset = ["dEQP-GL3.info.*", "dEQP-GL3.functional.*"], 426 exclude = [ 427 "dEQP-GL3.functional.shaders.loops.*while*unconditional_continue*", 428 "dEQP-GL3.functional.shaders.loops.*while*only_continue*", 429 "dEQP-GL3.functional.shaders.loops.*while*double_continue*", 430 ], 431 ignore = [ 432 "dEQP-GL3.functional.transform_feedback.*" 433 ] 434 ), 435 NightlyRunConfig( 436 name = "arm_gles3_emu_x32_egl", 437 buildConfig = ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG, 438 generator = DEFAULT_WIN32_GENERATOR, 439 binaryName = "modules/egl/deqp-egl", 440 args = BASE_ARGS, 441 testset = ["dEQP-EGL.info.*", "dEQP-EGL.functional.*"], 442 exclude = [ 443 "dEQP-EGL.functional.sharing.gles2.multithread.*", 444 "dEQP-EGL.functional.multithread.*", 445 ], 446 ignore = [] 447 ), 448 NightlyRunConfig( 449 name = "opencl_x64_release", 450 buildConfig = NightlyBuildConfig("opencl_x64_release", "Release", ['-DDEQP_TARGET=opencl_icd']), 451 generator = DEFAULT_WIN64_GENERATOR, 452 binaryName = "modules/opencl/deqp-opencl", 453 args = ['--deqp-cl-platform-id=2 --deqp-cl-device-ids=1'] + BASE_ARGS, 454 testset = ["dEQP-CL.*"], 455 exclude = ["dEQP-CL.performance.*", "dEQP-CL.robustness.*", "dEQP-CL.stress.memory.*"], 456 ignore = [ 457 "dEQP-CL.scheduler.random.*", 458 "dEQP-CL.language.set_kernel_arg.random_structs.*", 459 "dEQP-CL.language.builtin_function.work_item.invalid_get_global_offset", 460 "dEQP-CL.language.call_function.arguments.random_structs.*", 461 "dEQP-CL.language.call_kernel.random_structs.*", 462 "dEQP-CL.language.inf_nan.nan.frexp.float", 463 "dEQP-CL.language.inf_nan.nan.lgamma_r.float", 464 "dEQP-CL.language.inf_nan.nan.modf.float", 465 "dEQP-CL.language.inf_nan.nan.sqrt.float", 466 "dEQP-CL.api.multithread.*", 467 "dEQP-CL.api.callback.random.nested.*", 468 "dEQP-CL.api.memory_migration.out_of_order_host.image2d.single_device_kernel_migrate_validate_abb", 469 "dEQP-CL.api.memory_migration.out_of_order.image2d.single_device_kernel_migrate_kernel_validate_abbb", 470 "dEQP-CL.image.addressing_filtering12.1d_array.*", 471 "dEQP-CL.image.addressing_filtering12.2d_array.*" 472 ] 473 ) 474 ] 475 476 if __name__ == "__main__": 477 config = None 478 479 if len(sys.argv) == 2: 480 cfgName = sys.argv[1] 481 for curCfg in CONFIGS: 482 if curCfg.name == cfgName: 483 config = curCfg 484 break 485 486 if config != None: 487 isOk = runNightly(config) 488 if not isOk: 489 sys.exit(-1) 490 else: 491 print "%s: [config]" % sys.argv[0] 492 print "" 493 print " Available configs:" 494 for config in CONFIGS: 495 print " %s" % config.name 496 sys.exit(-1) 497
