1 /*------------------------------------------------------------------------- 2 * OpenGL Conformance Test Suite 3 * ----------------------------- 4 * 5 * Copyright (c) 2016 Google Inc. 6 * Copyright (c) 2016 The Khronos Group Inc. 7 * 8 * Licensed under the Apache License, Version 2.0 (the "License"); 9 * you may not use this file except in compliance with the License. 10 * You may obtain a copy of the License at 11 * 12 * http://www.apache.org/licenses/LICENSE-2.0 13 * 14 * Unless required by applicable law or agreed to in writing, software 15 * distributed under the License is distributed on an "AS IS" BASIS, 16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 * See the License for the specific language governing permissions and 18 * limitations under the License. 19 * 20 */ /*! 21 * \file 22 * \brief CTS runner. 23 */ /*-------------------------------------------------------------------*/ 24 25 #include "glcTestRunner.hpp" 26 #include "deFilePath.hpp" 27 #include "deStringUtil.hpp" 28 #include "deUniquePtr.hpp" 29 #include "glcConfigList.hpp" 30 #include "qpXmlWriter.h" 31 #include "tcuApp.hpp" 32 #include "tcuCommandLine.hpp" 33 #include "tcuTestLog.hpp" 34 #include "tcuTestSessionExecutor.hpp" 35 36 #include <iterator> 37 38 namespace glcts 39 { 40 41 using std::vector; 42 using std::string; 43 44 // RunSession 45 46 class RunSession 47 { 48 public: 49 RunSession(tcu::Platform& platform, tcu::Archive& archive, const int numArgs, const char* const* args) 50 : m_cmdLine(numArgs, args) 51 , m_log(m_cmdLine.getLogFileName(), m_cmdLine.getLogFlags()) 52 , m_app(platform, archive, m_log, m_cmdLine) 53 { 54 } 55 56 inline bool iterate(void) 57 { 58 return m_app.iterate(); 59 } 60 61 inline const tcu::TestRunStatus& getResult(void) const 62 { 63 return m_app.getResult(); 64 } 65 66 private: 67 tcu::CommandLine m_cmdLine; 68 tcu::TestLog m_log; 69 tcu::App m_app; 70 }; 71 72 static void appendConfigArgs(const Config& config, std::vector<std::string>& args, const char* fboConfig) 73 { 74 if (fboConfig != NULL) 75 { 76 args.push_back(string("--deqp-gl-config-name=") + fboConfig); 77 args.push_back("--deqp-surface-type=fbo"); 78 } 79 80 if (config.type != CONFIGTYPE_DEFAULT) 81 { 82 // \todo [2013-05-06 pyry] Test all surface types for some configs? 83 if (fboConfig == NULL) 84 { 85 if (config.surfaceTypes & SURFACETYPE_WINDOW) 86 args.push_back("--deqp-surface-type=window"); 87 else if (config.surfaceTypes & SURFACETYPE_PBUFFER) 88 args.push_back("--deqp-surface-type=pbuffer"); 89 else if (config.surfaceTypes & SURFACETYPE_PIXMAP) 90 args.push_back("--deqp-surface-type=pixmap"); 91 } 92 93 args.push_back(string("--deqp-gl-config-id=") + de::toString(config.id)); 94 95 if (config.type == CONFIGTYPE_EGL) 96 args.push_back("--deqp-gl-context-type=egl"); 97 else if (config.type == CONFIGTYPE_WGL) 98 args.push_back("--deqp-gl-context-type=wgl"); 99 } 100 } 101 102 typedef struct configInfo 103 { 104 deInt32 redBits; 105 deInt32 greenBits; 106 deInt32 blueBits; 107 deInt32 alphaBits; 108 deInt32 depthBits; 109 deInt32 stencilBits; 110 deInt32 samples; 111 } configInfo; 112 113 static configInfo parseConfigBitsFromName(const char* configName) 114 { 115 configInfo cfgInfo; 116 static const struct 117 { 118 const char* name; 119 int redBits; 120 int greenBits; 121 int blueBits; 122 int alphaBits; 123 } colorCfgs[] = { 124 { "rgba8888", 8, 8, 8, 8 }, { "rgb565", 5, 6, 5, 0 }, 125 }; 126 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(colorCfgs); ndx++) 127 { 128 if (!strncmp(configName, colorCfgs[ndx].name, strlen(colorCfgs[ndx].name))) 129 { 130 cfgInfo.redBits = colorCfgs[ndx].redBits; 131 cfgInfo.greenBits = colorCfgs[ndx].greenBits; 132 cfgInfo.blueBits = colorCfgs[ndx].blueBits; 133 cfgInfo.alphaBits = colorCfgs[ndx].alphaBits; 134 135 configName += strlen(colorCfgs[ndx].name); 136 break; 137 } 138 } 139 140 static const struct 141 { 142 const char* name; 143 int depthBits; 144 } depthCfgs[] = { 145 { "d0", 0 }, { "d24", 24 }, 146 }; 147 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(depthCfgs); ndx++) 148 { 149 if (!strncmp(configName, depthCfgs[ndx].name, strlen(depthCfgs[ndx].name))) 150 { 151 cfgInfo.depthBits = depthCfgs[ndx].depthBits; 152 153 configName += strlen(depthCfgs[ndx].name); 154 break; 155 } 156 } 157 158 static const struct 159 { 160 const char* name; 161 int stencilBits; 162 } stencilCfgs[] = { 163 { "s0", 0 }, { "s8", 8 }, 164 }; 165 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(stencilCfgs); ndx++) 166 { 167 if (!strncmp(configName, stencilCfgs[ndx].name, strlen(stencilCfgs[ndx].name))) 168 { 169 cfgInfo.stencilBits = stencilCfgs[ndx].stencilBits; 170 171 configName += strlen(stencilCfgs[ndx].name); 172 break; 173 } 174 } 175 176 static const struct 177 { 178 const char* name; 179 int samples; 180 } multiSampleCfgs[] = { 181 { "ms0", 0 }, { "ms4", 4 }, 182 }; 183 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(multiSampleCfgs); ndx++) 184 { 185 if (!strncmp(configName, multiSampleCfgs[ndx].name, strlen(multiSampleCfgs[ndx].name))) 186 { 187 cfgInfo.samples = multiSampleCfgs[ndx].samples; 188 189 configName += strlen(multiSampleCfgs[ndx].name); 190 break; 191 } 192 } 193 194 return cfgInfo; 195 } 196 197 static const char* getApiName(glu::ApiType apiType) 198 { 199 if (apiType == glu::ApiType::es(2, 0)) 200 return "gles2"; 201 else if (apiType == glu::ApiType::es(3, 0)) 202 return "gles3"; 203 else if (apiType == glu::ApiType::es(3, 1)) 204 return "gles31"; 205 else if (apiType == glu::ApiType::es(3, 2)) 206 return "gles32"; 207 else if (apiType == glu::ApiType::core(3, 0)) 208 return "gl30"; 209 else if (apiType == glu::ApiType::core(3, 1)) 210 return "gl31"; 211 else if (apiType == glu::ApiType::core(3, 2)) 212 return "gl32"; 213 else if (apiType == glu::ApiType::core(3, 3)) 214 return "gl33"; 215 else if (apiType == glu::ApiType::core(4, 0)) 216 return "gl40"; 217 else if (apiType == glu::ApiType::core(4, 1)) 218 return "gl41"; 219 else if (apiType == glu::ApiType::core(4, 2)) 220 return "gl42"; 221 else if (apiType == glu::ApiType::core(4, 3)) 222 return "gl43"; 223 else if (apiType == glu::ApiType::core(4, 4)) 224 return "gl44"; 225 else if (apiType == glu::ApiType::core(4, 5)) 226 return "gl45"; 227 else if (apiType == glu::ApiType::core(4, 6)) 228 return "gl46"; 229 else 230 throw std::runtime_error("Unknown context type"); 231 } 232 233 static const string getCaseListFileOption(const char* mustpassDir, const char* apiName, const char* mustpassName) 234 { 235 #if DE_OS == DE_OS_ANDROID 236 const string case_list_option = "--deqp-caselist-resource="; 237 #else 238 const string case_list_option = "--deqp-caselist-file="; 239 #endif 240 return case_list_option + mustpassDir + apiName + "-" + mustpassName + ".txt"; 241 } 242 243 static const string getLogFileName(const char* apiName, const char* configName, const int iterId, const int runId, 244 const int width, const int height, const int seed) 245 { 246 string res = string("config-") + apiName + "-" + configName + "-cfg-" + de::toString(iterId) + "-run-" + 247 de::toString(runId) + "-width-" + de::toString(width) + "-height-" + de::toString(height); 248 if (seed != -1) 249 { 250 res += "-seed-" + de::toString(seed); 251 } 252 res += ".qpa"; 253 254 return res; 255 } 256 257 static void getBaseOptions(std::vector<std::string>& args, const char* mustpassDir, const char* apiName, 258 const char* configName, const char* screenRotation, int width, int height) 259 { 260 args.push_back(getCaseListFileOption(mustpassDir, apiName, configName)); 261 args.push_back(string("--deqp-screen-rotation=") + screenRotation); 262 args.push_back(string("--deqp-surface-width=") + de::toString(width)); 263 args.push_back(string("--deqp-surface-height=") + de::toString(height)); 264 args.push_back("--deqp-watchdog=disable"); 265 } 266 267 static bool isGLConfigCompatible(configInfo cfgInfo, const AOSPConfig& config) 268 { 269 return cfgInfo.redBits == config.redBits && cfgInfo.greenBits == config.greenBits && 270 cfgInfo.blueBits == config.blueBits && cfgInfo.alphaBits == config.alphaBits && 271 cfgInfo.depthBits == config.depthBits && cfgInfo.stencilBits == config.stencilBits && 272 cfgInfo.samples == config.samples; 273 } 274 275 static void getTestRunsForAOSPEGL(vector<TestRunParams>& runs, const ConfigList& configs) 276 { 277 #include "glcAospMustpassEgl.hpp" 278 279 for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_egl_first_cfg); ++i) 280 { 281 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_egl_first_cfg[i].glConfigName); 282 283 vector<AOSPConfig>::const_iterator cfgIter; 284 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter) 285 { 286 // find first compatible config 287 if ((*cfgIter).type == CONFIGTYPE_EGL && isGLConfigCompatible(cfgInfo, *cfgIter)) 288 { 289 break; 290 } 291 } 292 293 if (cfgIter == configs.aospConfigs.end()) 294 { 295 // No suitable configuration found. Skipping EGL tests 296 continue; 297 } 298 299 const char* apiName = "egl"; 300 301 const int width = aosp_mustpass_egl_first_cfg[i].surfaceWidth; 302 const int height = aosp_mustpass_egl_first_cfg[i].surfaceHeight; 303 304 TestRunParams params; 305 params.logFilename = 306 getLogFileName(apiName, aosp_mustpass_egl_first_cfg[i].configName, 1, i, width, height, -1); 307 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_egl_first_cfg[i].configName, 308 aosp_mustpass_egl_first_cfg[i].screenRotation, width, height); 309 310 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_egl_first_cfg[i].glConfigName)); 311 312 runs.push_back(params); 313 } 314 } 315 316 static void getTestRunsForAOSPES(vector<TestRunParams>& runs, const ConfigList& configs, const glu::ApiType apiType) 317 { 318 #include "glcAospMustpassEs.hpp" 319 320 for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_es_first_cfg); ++i) 321 { 322 if (!glu::contextSupports(glu::ContextType(apiType), aosp_mustpass_es_first_cfg[i].apiType)) 323 continue; 324 325 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_es_first_cfg[i].glConfigName); 326 327 vector<AOSPConfig>::const_iterator cfgIter; 328 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter) 329 { 330 // find first compatible config 331 if (isGLConfigCompatible(cfgInfo, *cfgIter)) 332 { 333 break; 334 } 335 } 336 337 if (cfgIter == configs.aospConfigs.end()) 338 { 339 TCU_FAIL(("No suitable configuration found for GL config " + 340 de::toString(aosp_mustpass_es_first_cfg[i].glConfigName)) 341 .c_str()); 342 return; 343 } 344 345 const char* apiName = getApiName(aosp_mustpass_es_first_cfg[i].apiType); 346 347 const int width = aosp_mustpass_es_first_cfg[i].surfaceWidth; 348 const int height = aosp_mustpass_es_first_cfg[i].surfaceHeight; 349 350 TestRunParams params; 351 params.logFilename = getLogFileName(apiName, aosp_mustpass_es_first_cfg[i].configName, 1, i, width, height, -1); 352 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_es_first_cfg[i].configName, 353 aosp_mustpass_es_first_cfg[i].screenRotation, width, height); 354 355 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_es_first_cfg[i].glConfigName)); 356 357 //set surface type 358 if ((*cfgIter).surfaceTypes & SURFACETYPE_WINDOW) 359 params.args.push_back("--deqp-surface-type=window"); 360 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PBUFFER) 361 params.args.push_back("--deqp-surface-type=pbuffer"); 362 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PIXMAP) 363 params.args.push_back("--deqp-surface-type=pixmap"); 364 runs.push_back(params); 365 } 366 } 367 368 static void getTestRunsForNoContext(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs, const RunParams* runParams, 369 const int numRunParams, const char* mustpassDir) 370 { 371 vector<Config>::const_iterator cfgIter = configs.configs.begin(); 372 373 for (int i = 0; i < numRunParams; ++i) 374 { 375 if (!glu::contextSupports(glu::ContextType(type), runParams[i].apiType)) 376 continue; 377 378 const char* apiName = getApiName(runParams[i].apiType); 379 380 const int width = runParams[i].surfaceWidth; 381 const int height = runParams[i].surfaceHeight; 382 const int seed = runParams[i].baseSeed; 383 384 TestRunParams params; 385 params.logFilename = getLogFileName(apiName, runParams[i].configName, 1, i, width, height, seed); 386 387 getBaseOptions(params.args, mustpassDir, apiName, runParams[i].configName, runParams[i].screenRotation, width, 388 height); 389 390 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed)); 391 392 appendConfigArgs(*cfgIter, params.args, runParams[i].fboConfig); 393 394 runs.push_back(params); 395 } 396 } 397 398 static void getTestRunsForNoContextES(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs) 399 { 400 #include "glcKhronosMustpassEsNocontext.hpp" 401 getTestRunsForNoContext(type, runs, configs, khronos_mustpass_es_nocontext_first_cfg, 402 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_nocontext_first_cfg), mustpassDir); 403 } 404 405 static void getTestRunsForES(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs) 406 { 407 getTestRunsForAOSPEGL(runs, configs); 408 getTestRunsForAOSPES(runs, configs, type); 409 getTestRunsForNoContextES(type, runs, configs); 410 411 #include "glcKhronosMustpassEs.hpp" 412 413 for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter) 414 { 415 const bool isFirst = cfgIter == configs.configs.begin(); 416 const int numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_es_first_cfg) : 417 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_other_cfg); 418 const RunParams* runParams = isFirst ? khronos_mustpass_es_first_cfg : khronos_mustpass_es_other_cfg; 419 420 for (int runNdx = 0; runNdx < numRunParams; runNdx++) 421 { 422 if (!glu::contextSupports(glu::ContextType(type), runParams[runNdx].apiType)) 423 continue; 424 425 const char* apiName = getApiName(runParams[runNdx].apiType); 426 427 const int width = runParams[runNdx].surfaceWidth; 428 const int height = runParams[runNdx].surfaceHeight; 429 const int seed = runParams[runNdx].baseSeed; 430 431 TestRunParams params; 432 433 params.logFilename = 434 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed); 435 436 getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName, 437 runParams[runNdx].screenRotation, width, height); 438 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed)); 439 440 appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig); 441 442 runs.push_back(params); 443 } 444 } 445 } 446 447 static void getTestRunsForNoContextGL(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs) 448 { 449 #include "glcKhronosMustpassGlNocontext.hpp" 450 getTestRunsForNoContext(type, runs, configs, khronos_mustpass_gl_nocontext_first_cfg, 451 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_nocontext_first_cfg), mustpassDir); 452 } 453 454 static void getTestRunsForGL(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs) 455 { 456 getTestRunsForNoContextGL(type, runs, configs); 457 #include "glcKhronosMustpassGl.hpp" 458 459 for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter) 460 { 461 const bool isFirst = cfgIter == configs.configs.begin(); 462 const int numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_first_cfg) : 463 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_other_cfg); 464 const RunParams* runParams = isFirst ? khronos_mustpass_gl_first_cfg : khronos_mustpass_gl_other_cfg; 465 466 for (int runNdx = 0; runNdx < numRunParams; runNdx++) 467 { 468 if (type != runParams[runNdx].apiType) 469 continue; 470 471 const char* apiName = getApiName(runParams[runNdx].apiType); 472 473 const int width = runParams[runNdx].surfaceWidth; 474 const int height = runParams[runNdx].surfaceHeight; 475 const int seed = runParams[runNdx].baseSeed; 476 477 TestRunParams params; 478 479 params.logFilename = 480 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed); 481 482 getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName, 483 runParams[runNdx].screenRotation, width, height); 484 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed)); 485 486 appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig); 487 488 runs.push_back(params); 489 } 490 } 491 } 492 493 static void getTestRunParams(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs) 494 { 495 switch (type.getProfile()) 496 { 497 case glu::PROFILE_CORE: 498 getTestRunsForGL(type, configs, runs); 499 break; 500 case glu::PROFILE_ES: 501 getTestRunsForES(type, configs, runs); 502 break; 503 default: 504 throw std::runtime_error("Unknown context type"); 505 } 506 } 507 508 struct FileDeleter 509 { 510 void operator()(FILE* file) const 511 { 512 if (file) 513 fclose(file); 514 } 515 }; 516 517 struct XmlWriterDeleter 518 { 519 void operator()(qpXmlWriter* writer) const 520 { 521 if (writer) 522 qpXmlWriter_destroy(writer); 523 } 524 }; 525 526 static const char* getRunTypeName(glu::ApiType type) 527 { 528 if (type == glu::ApiType::es(2, 0)) 529 return "es2"; 530 else if (type == glu::ApiType::es(3, 0)) 531 return "es3"; 532 else if (type == glu::ApiType::es(3, 1)) 533 return "es31"; 534 else if (type == glu::ApiType::es(3, 2)) 535 return "es32"; 536 else if (type == glu::ApiType::core(3, 0)) 537 return "gl30"; 538 else if (type == glu::ApiType::core(3, 1)) 539 return "gl31"; 540 else if (type == glu::ApiType::core(3, 2)) 541 return "gl32"; 542 else if (type == glu::ApiType::core(3, 3)) 543 return "gl33"; 544 else if (type == glu::ApiType::core(4, 0)) 545 return "gl40"; 546 else if (type == glu::ApiType::core(4, 1)) 547 return "gl41"; 548 else if (type == glu::ApiType::core(4, 2)) 549 return "gl42"; 550 else if (type == glu::ApiType::core(4, 3)) 551 return "gl43"; 552 else if (type == glu::ApiType::core(4, 4)) 553 return "gl44"; 554 else if (type == glu::ApiType::core(4, 5)) 555 return "gl45"; 556 else if (type == glu::ApiType::core(4, 6)) 557 return "gl46"; 558 else 559 return DE_NULL; 560 } 561 562 #define XML_CHECK(X) \ 563 if (!(X)) \ 564 throw tcu::Exception("Writing XML failed") 565 566 static void writeRunSummary(const TestRunSummary& summary, const char* filename) 567 { 568 de::UniquePtr<FILE, FileDeleter> out(fopen(filename, "wb")); 569 if (!out) 570 throw tcu::Exception(string("Failed to open ") + filename); 571 572 de::UniquePtr<qpXmlWriter, XmlWriterDeleter> writer(qpXmlWriter_createFileWriter(out.get(), DE_FALSE, DE_FALSE)); 573 if (!writer) 574 throw std::bad_alloc(); 575 576 XML_CHECK(qpXmlWriter_startDocument(writer.get())); 577 578 { 579 qpXmlAttribute attribs[2]; 580 581 attribs[0] = qpSetStringAttrib("Type", getRunTypeName(summary.runType)); 582 attribs[1] = qpSetBoolAttrib("Conformant", summary.isConformant ? DE_TRUE : DE_FALSE); 583 584 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Summary", DE_LENGTH_OF_ARRAY(attribs), attribs)); 585 } 586 587 // Config run 588 { 589 qpXmlAttribute attribs[1]; 590 attribs[0] = qpSetStringAttrib("FileName", summary.configLogFilename.c_str()); 591 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Configs", DE_LENGTH_OF_ARRAY(attribs), attribs) && 592 qpXmlWriter_endElement(writer.get(), "Configs")); 593 } 594 595 // Record test run parameters (log filename & command line). 596 for (vector<TestRunParams>::const_iterator runIter = summary.runParams.begin(); runIter != summary.runParams.end(); 597 ++runIter) 598 { 599 string cmdLine; 600 qpXmlAttribute attribs[2]; 601 602 for (vector<string>::const_iterator argIter = runIter->args.begin(); argIter != runIter->args.end(); ++argIter) 603 { 604 if (argIter != runIter->args.begin()) 605 cmdLine += " "; 606 cmdLine += *argIter; 607 } 608 609 attribs[0] = qpSetStringAttrib("FileName", runIter->logFilename.c_str()); 610 attribs[1] = qpSetStringAttrib("CmdLine", cmdLine.c_str()); 611 612 XML_CHECK(qpXmlWriter_startElement(writer.get(), "TestRun", DE_LENGTH_OF_ARRAY(attribs), attribs) && 613 qpXmlWriter_endElement(writer.get(), "TestRun")); 614 } 615 616 XML_CHECK(qpXmlWriter_endElement(writer.get(), "Summary")); 617 XML_CHECK(qpXmlWriter_endDocument(writer.get())); 618 } 619 620 #undef XML_CHECK 621 622 TestRunner::TestRunner(tcu::Platform& platform, tcu::Archive& archive, const char* logDirPath, glu::ApiType type, 623 deUint32 flags) 624 : m_platform(platform) 625 , m_archive(archive) 626 , m_logDirPath(logDirPath) 627 , m_type(type) 628 , m_flags(flags) 629 , m_iterState(ITERATE_INIT) 630 , m_curSession(DE_NULL) 631 , m_sessionsExecuted(0) 632 , m_sessionsPassed(0) 633 , m_sessionsFailed(0) 634 { 635 } 636 637 TestRunner::~TestRunner(void) 638 { 639 delete m_curSession; 640 } 641 642 bool TestRunner::iterate(void) 643 { 644 switch (m_iterState) 645 { 646 case ITERATE_INIT: 647 init(); 648 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT; 649 return true; 650 651 case ITERATE_DEINIT: 652 deinit(); 653 m_iterState = ITERATE_INIT; 654 return false; 655 656 case ITERATE_INIT_SESSION: 657 DE_ASSERT(m_sessionIter != m_runSessions.end()); 658 initSession(*m_sessionIter); 659 if (m_flags & PRINT_SUMMARY) 660 m_iterState = ITERATE_DEINIT_SESSION; 661 else 662 m_iterState = ITERATE_ITERATE_SESSION; 663 return true; 664 665 case ITERATE_DEINIT_SESSION: 666 deinitSession(); 667 ++m_sessionIter; 668 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT; 669 return true; 670 671 case ITERATE_ITERATE_SESSION: 672 if (!iterateSession()) 673 m_iterState = ITERATE_DEINIT_SESSION; 674 return true; 675 676 default: 677 DE_ASSERT(false); 678 return false; 679 } 680 } 681 682 void TestRunner::init(void) 683 { 684 DE_ASSERT(m_runSessions.empty() && m_summary.runParams.empty()); 685 686 tcu::print("Running %s conformance\n", glu::getApiTypeDescription(m_type)); 687 688 m_summary.runType = m_type; 689 690 // Get list of configs to test. 691 ConfigList configList; 692 getDefaultConfigList(m_platform, m_type, configList); 693 694 tcu::print(" found %d compatible and %d excluded configs\n", (int)configList.configs.size(), 695 (int)configList.excludedConfigs.size()); 696 697 // Config list run. 698 { 699 const char* configLogFilename = "configs.qpa"; 700 TestRunParams configRun; 701 702 configRun.logFilename = configLogFilename; 703 configRun.args.push_back("--deqp-case=CTS-Configs.*"); 704 m_runSessions.push_back(configRun); 705 706 m_summary.configLogFilename = configLogFilename; 707 } 708 709 // Conformance test type specific runs 710 getTestRunParams(m_type, configList, m_runSessions); 711 712 // Record run params for summary. 713 for (std::vector<TestRunParams>::const_iterator runIter = m_runSessions.begin() + 1; runIter != m_runSessions.end(); 714 ++runIter) 715 m_summary.runParams.push_back(*runIter); 716 717 // Session iterator 718 m_sessionIter = m_runSessions.begin(); 719 } 720 721 void TestRunner::deinit(void) 722 { 723 // Print out totals. 724 bool isConformant = m_sessionsExecuted == m_sessionsPassed; 725 DE_ASSERT(m_sessionsExecuted == m_sessionsPassed + m_sessionsFailed); 726 tcu::print("\n%d/%d sessions passed, conformance test %s\n", m_sessionsPassed, m_sessionsExecuted, 727 isConformant ? "PASSED" : "FAILED"); 728 729 m_summary.isConformant = isConformant; 730 731 // Write out summary. 732 writeRunSummary(m_summary, de::FilePath::join(m_logDirPath, "cts-run-summary.xml").getPath()); 733 734 m_runSessions.clear(); 735 m_summary.clear(); 736 } 737 738 void TestRunner::initSession(const TestRunParams& runParams) 739 { 740 DE_ASSERT(!m_curSession); 741 742 tcu::print("\n Test run %d / %d\n", (int)(m_sessionIter - m_runSessions.begin() + 1), (int)m_runSessions.size()); 743 744 // Compute final args for run. 745 vector<string> args(runParams.args); 746 args.push_back(string("--deqp-log-filename=") + de::FilePath::join(m_logDirPath, runParams.logFilename).getPath()); 747 748 if (!(m_flags & VERBOSE_IMAGES)) 749 args.push_back("--deqp-log-images=disable"); 750 751 if (!(m_flags & VERBOSE_SHADERS)) 752 args.push_back("--deqp-log-shader-sources=disable"); 753 754 std::ostringstream ostr; 755 std::ostream_iterator<string> out_it(ostr, ", "); 756 std::copy(args.begin(), args.end(), out_it); 757 tcu::print("\n Config: %s \n\n", ostr.str().c_str()); 758 759 // Translate to argc, argv 760 vector<const char*> argv; 761 argv.push_back("cts-runner"); // Dummy binary name 762 for (vector<string>::const_iterator i = args.begin(); i != args.end(); i++) 763 argv.push_back(i->c_str()); 764 765 // Create session 766 m_curSession = new RunSession(m_platform, m_archive, (int)argv.size(), &argv[0]); 767 } 768 769 void TestRunner::deinitSession(void) 770 { 771 DE_ASSERT(m_curSession); 772 773 // Collect results. 774 // \note NotSupported is treated as pass. 775 const tcu::TestRunStatus& result = m_curSession->getResult(); 776 bool isOk = 777 result.numExecuted == (result.numPassed + result.numNotSupported + result.numWarnings) && result.isComplete; 778 779 DE_ASSERT(result.numExecuted == result.numPassed + result.numFailed + result.numNotSupported + result.numWarnings); 780 781 m_sessionsExecuted += 1; 782 (isOk ? m_sessionsPassed : m_sessionsFailed) += 1; 783 784 delete m_curSession; 785 m_curSession = DE_NULL; 786 } 787 788 inline bool TestRunner::iterateSession(void) 789 { 790 return m_curSession->iterate(); 791 } 792 793 } // glcts 794
