1 /*------------------------------------------------------------------------- 2 * drawElements Quality Program Test Executor 3 * ------------------------------------------ 4 * 5 * Copyright 2014 The Android Open Source Project 6 * 7 * Licensed under the Apache License, Version 2.0 (the "License"); 8 * you may not use this file except in compliance with the License. 9 * You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, software 14 * distributed under the License is distributed on an "AS IS" BASIS, 15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 * See the License for the specific language governing permissions and 17 * limitations under the License. 18 * 19 *//*! 20 * \file 21 * \brief Test log compare utility. 22 *//*--------------------------------------------------------------------*/ 23 24 #include "xeTestLogParser.hpp" 25 #include "xeTestResultParser.hpp" 26 #include "deFilePath.hpp" 27 #include "deString.h" 28 #include "deThread.hpp" 29 #include "deCommandLine.hpp" 30 31 #include <vector> 32 #include <string> 33 #include <cstdio> 34 #include <cstdlib> 35 #include <fstream> 36 #include <iostream> 37 #include <set> 38 #include <map> 39 40 using std::vector; 41 using std::string; 42 using std::set; 43 using std::map; 44 45 enum OutputMode 46 { 47 OUTPUTMODE_ALL = 0, 48 OUTPUTMODE_DIFF, 49 50 OUTPUTMODE_LAST 51 }; 52 53 enum OutputFormat 54 { 55 OUTPUTFORMAT_TEXT = 0, 56 OUTPUTFORMAT_CSV, 57 58 OUTPUTFORMAT_LAST 59 }; 60 61 enum OutputValue 62 { 63 OUTPUTVALUE_STATUS_CODE = 0, 64 OUTPUTVALUE_STATUS_DETAILS, 65 66 OUTPUTVALUE_LAST 67 }; 68 69 namespace opt 70 { 71 72 DE_DECLARE_COMMAND_LINE_OPT(OutMode, OutputMode); 73 DE_DECLARE_COMMAND_LINE_OPT(OutFormat, OutputFormat); 74 DE_DECLARE_COMMAND_LINE_OPT(OutValue, OutputValue); 75 76 static void registerOptions (de::cmdline::Parser& parser) 77 { 78 using de::cmdline::Option; 79 using de::cmdline::NamedValue; 80 81 static const NamedValue<OutputMode> s_outputModes[] = 82 { 83 { "all", OUTPUTMODE_ALL }, 84 { "diff", OUTPUTMODE_DIFF } 85 }; 86 static const NamedValue<OutputFormat> s_outputFormats[] = 87 { 88 { "text", OUTPUTFORMAT_TEXT }, 89 { "csv", OUTPUTFORMAT_CSV } 90 }; 91 static const NamedValue<OutputValue> s_outputValues[] = 92 { 93 { "code", OUTPUTVALUE_STATUS_CODE }, 94 { "details", OUTPUTVALUE_STATUS_DETAILS } 95 }; 96 97 parser << Option<OutFormat> ("f", "format", "Output format", s_outputFormats, "csv") 98 << Option<OutMode> ("m", "mode", "Output mode", s_outputModes, "all") 99 << Option<OutValue> ("v", "value", "Value to extract", s_outputValues, "code"); 100 } 101 102 } // opt 103 104 struct CommandLine 105 { 106 CommandLine (void) 107 : outMode (OUTPUTMODE_ALL) 108 , outFormat (OUTPUTFORMAT_CSV) 109 , outValue (OUTPUTVALUE_STATUS_CODE) 110 { 111 } 112 113 OutputMode outMode; 114 OutputFormat outFormat; 115 OutputValue outValue; 116 vector<string> filenames; 117 }; 118 119 struct ShortBatchResult 120 { 121 vector<xe::TestCaseResultHeader> resultHeaders; 122 map<string, int> resultMap; 123 }; 124 125 class ShortResultHandler : public xe::TestLogHandler 126 { 127 public: 128 ShortResultHandler (ShortBatchResult& result) 129 : m_result(result) 130 { 131 } 132 133 void setSessionInfo (const xe::SessionInfo&) 134 { 135 // Ignored. 136 } 137 138 xe::TestCaseResultPtr startTestCaseResult (const char* casePath) 139 { 140 return xe::TestCaseResultPtr(new xe::TestCaseResultData(casePath)); 141 } 142 143 void testCaseResultUpdated (const xe::TestCaseResultPtr&) 144 { 145 // Ignored. 146 } 147 148 void testCaseResultComplete (const xe::TestCaseResultPtr& caseData) 149 { 150 xe::TestCaseResultHeader header; 151 int caseNdx = (int)m_result.resultHeaders.size(); 152 153 header.casePath = caseData->getTestCasePath(); 154 header.caseType = xe::TESTCASETYPE_SELF_VALIDATE; 155 header.statusCode = caseData->getStatusCode(); 156 header.statusDetails = caseData->getStatusDetails(); 157 158 if (header.statusCode == xe::TESTSTATUSCODE_LAST) 159 { 160 xe::TestCaseResult fullResult; 161 162 xe::parseTestCaseResultFromData(&m_testResultParser, &fullResult, *caseData.get()); 163 164 header = xe::TestCaseResultHeader(fullResult); 165 } 166 167 // Insert into result list & map. 168 m_result.resultHeaders.push_back(header); 169 m_result.resultMap[header.casePath] = caseNdx; 170 } 171 172 private: 173 ShortBatchResult& m_result; 174 xe::TestResultParser m_testResultParser; 175 }; 176 177 static void readLogFile (ShortBatchResult& batchResult, const char* filename) 178 { 179 std::ifstream in (filename, std::ifstream::binary|std::ifstream::in); 180 ShortResultHandler resultHandler (batchResult); 181 xe::TestLogParser parser (&resultHandler); 182 deUint8 buf [1024]; 183 int numRead = 0; 184 185 for (;;) 186 { 187 in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf)); 188 numRead = (int)in.gcount(); 189 190 if (numRead <= 0) 191 break; 192 193 parser.parse(&buf[0], numRead); 194 } 195 196 in.close(); 197 } 198 199 class LogFileReader : public de::Thread 200 { 201 public: 202 LogFileReader (ShortBatchResult& batchResult, const char* filename) 203 : m_batchResult (batchResult) 204 , m_filename (filename) 205 { 206 } 207 208 void run (void) 209 { 210 readLogFile(m_batchResult, m_filename.c_str()); 211 } 212 213 private: 214 ShortBatchResult& m_batchResult; 215 std::string m_filename; 216 }; 217 218 static void computeCaseList (vector<string>& cases, const vector<ShortBatchResult>& batchResults) 219 { 220 // \todo [2012-07-10 pyry] Do proper case ordering (eg. handle missing cases nicely). 221 set<string> addedCases; 222 223 for (vector<ShortBatchResult>::const_iterator batchIter = batchResults.begin(); batchIter != batchResults.end(); batchIter++) 224 { 225 for (vector<xe::TestCaseResultHeader>::const_iterator caseIter = batchIter->resultHeaders.begin(); caseIter != batchIter->resultHeaders.end(); caseIter++) 226 { 227 if (addedCases.find(caseIter->casePath) == addedCases.end()) 228 { 229 cases.push_back(caseIter->casePath); 230 addedCases.insert(caseIter->casePath); 231 } 232 } 233 } 234 } 235 236 static void getTestResultHeaders (vector<xe::TestCaseResultHeader>& headers, const vector<ShortBatchResult>& batchResults, const char* casePath) 237 { 238 headers.resize(batchResults.size()); 239 240 for (int ndx = 0; ndx < (int)batchResults.size(); ndx++) 241 { 242 const ShortBatchResult& batchResult = batchResults[ndx]; 243 map<string, int>::const_iterator resultPos = batchResult.resultMap.find(casePath); 244 245 if (resultPos != batchResult.resultMap.end()) 246 headers[ndx] = batchResult.resultHeaders[resultPos->second]; 247 else 248 { 249 headers[ndx].casePath = casePath; 250 headers[ndx].caseType = xe::TESTCASETYPE_SELF_VALIDATE; 251 headers[ndx].statusCode = xe::TESTSTATUSCODE_LAST; 252 } 253 } 254 } 255 256 static const char* getStatusCodeName (xe::TestStatusCode code) 257 { 258 if (code == xe::TESTSTATUSCODE_LAST) 259 return "Missing"; 260 else 261 return xe::getTestStatusCodeName(code); 262 } 263 264 static bool runCompare (const CommandLine& cmdLine, std::ostream& dst) 265 { 266 vector<ShortBatchResult> results; 267 vector<string> batchNames; 268 bool compareOk = true; 269 270 XE_CHECK(!cmdLine.filenames.empty()); 271 272 try 273 { 274 // Read in batch results 275 results.resize(cmdLine.filenames.size()); 276 { 277 std::vector<de::SharedPtr<LogFileReader> > readers; 278 279 for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++) 280 { 281 readers.push_back(de::SharedPtr<LogFileReader>(new LogFileReader(results[ndx], cmdLine.filenames[ndx].c_str()))); 282 readers.back()->start(); 283 } 284 285 for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++) 286 { 287 readers[ndx]->join(); 288 289 // Use file name as batch name. 290 batchNames.push_back(de::FilePath(cmdLine.filenames[ndx].c_str()).getBaseName()); 291 } 292 } 293 294 // Compute unified case list. 295 vector<string> caseList; 296 computeCaseList(caseList, results); 297 298 // Stats. 299 int numCases = (int)caseList.size(); 300 int numEqual = 0; 301 302 if (cmdLine.outFormat == OUTPUTFORMAT_CSV) 303 { 304 dst << "TestCasePath"; 305 for (vector<string>::const_iterator nameIter = batchNames.begin(); nameIter != batchNames.end(); nameIter++) 306 dst << "," << *nameIter; 307 dst << "\n"; 308 } 309 310 // Compare cases. 311 for (vector<string>::const_iterator caseIter = caseList.begin(); caseIter != caseList.end(); caseIter++) 312 { 313 const string& caseName = *caseIter; 314 vector<xe::TestCaseResultHeader> headers; 315 bool allEqual = true; 316 317 getTestResultHeaders(headers, results, caseName.c_str()); 318 319 for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin()+1; iter != headers.end(); iter++) 320 { 321 if (iter->statusCode != headers[0].statusCode) 322 { 323 allEqual = false; 324 break; 325 } 326 } 327 328 if (allEqual) 329 numEqual += 1; 330 331 if (cmdLine.outMode == OUTPUTMODE_ALL || !allEqual) 332 { 333 if (cmdLine.outFormat == OUTPUTFORMAT_TEXT) 334 { 335 dst << caseName << "\n"; 336 for (int ndx = 0; ndx < (int)headers.size(); ndx++) 337 dst << " " << batchNames[ndx] << ": " << getStatusCodeName(headers[ndx].statusCode) << " (" << headers[ndx].statusDetails << ")\n"; 338 dst << "\n"; 339 } 340 else if (cmdLine.outFormat == OUTPUTFORMAT_CSV) 341 { 342 dst << caseName; 343 for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin(); iter != headers.end(); iter++) 344 dst << "," << (cmdLine.outValue == OUTPUTVALUE_STATUS_CODE ? getStatusCodeName(iter->statusCode) : iter->statusDetails.c_str()); 345 dst << "\n"; 346 } 347 } 348 } 349 350 compareOk = numEqual == numCases; 351 352 if (cmdLine.outFormat == OUTPUTFORMAT_TEXT) 353 { 354 dst << " " << numEqual << " / " << numCases << " test case results match.\n"; 355 dst << " Comparison " << (compareOk ? "passed" : "FAILED") << "!\n"; 356 } 357 } 358 catch (const std::exception& e) 359 { 360 printf("%s\n", e.what()); 361 compareOk = false; 362 } 363 364 return compareOk; 365 } 366 367 static bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv) 368 { 369 de::cmdline::Parser parser; 370 de::cmdline::CommandLine opts; 371 372 XE_CHECK(argc >= 1); 373 374 opt::registerOptions(parser); 375 376 if (!parser.parse(argc-1, &argv[1], &opts, std::cerr) || 377 opts.getArgs().empty()) 378 { 379 std::cout << argv[0] << ": [options] [filenames]\n"; 380 parser.help(std::cout); 381 return false; 382 } 383 384 cmdLine.outFormat = opts.getOption<opt::OutFormat>(); 385 cmdLine.outMode = opts.getOption<opt::OutMode>(); 386 cmdLine.outValue = opts.getOption<opt::OutValue>(); 387 cmdLine.filenames = opts.getArgs(); 388 389 return true; 390 } 391 392 int main (int argc, const char* const* argv) 393 { 394 CommandLine cmdLine; 395 396 if (!parseCommandLine(cmdLine, argc, argv)) 397 return -1; 398 399 try 400 { 401 bool compareOk = runCompare(cmdLine, std::cout); 402 return compareOk ? 0 : -1; 403 } 404 catch (const std::exception& e) 405 { 406 printf("FATAL ERROR: %s\n", e.what()); 407 return -1; 408 } 409 } 410
