1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.tradefed.result; 17 18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 19 import com.android.compatibility.common.tradefed.testtype.CompatibilityTest; 20 import com.android.compatibility.common.tradefed.util.RetryType; 21 import com.android.compatibility.common.util.ChecksumReporter; 22 import com.android.compatibility.common.util.ICaseResult; 23 import com.android.compatibility.common.util.IInvocationResult; 24 import com.android.compatibility.common.util.IModuleResult; 25 import com.android.compatibility.common.util.ITestResult; 26 import com.android.compatibility.common.util.InvocationResult; 27 import com.android.compatibility.common.util.MetricsStore; 28 import com.android.compatibility.common.util.ReportLog; 29 import com.android.compatibility.common.util.ResultHandler; 30 import com.android.compatibility.common.util.ResultUploader; 31 import com.android.compatibility.common.util.TestStatus; 32 import com.android.ddmlib.Log.LogLevel; 33 import com.android.ddmlib.testrunner.TestIdentifier; 34 import com.android.tradefed.build.IBuildInfo; 35 import com.android.tradefed.config.Option; 36 import com.android.tradefed.config.Option.Importance; 37 import com.android.tradefed.config.OptionClass; 38 import com.android.tradefed.config.OptionCopier; 39 import com.android.tradefed.invoker.IInvocationContext; 40 import com.android.tradefed.log.LogUtil.CLog; 41 import com.android.tradefed.result.ILogSaver; 42 import com.android.tradefed.result.ILogSaverListener; 43 import com.android.tradefed.result.IShardableListener; 44 import com.android.tradefed.result.ITestInvocationListener; 45 import com.android.tradefed.result.ITestSummaryListener; 46 import com.android.tradefed.result.InputStreamSource; 47 import com.android.tradefed.result.LogDataType; 48 import com.android.tradefed.result.LogFile; 49 import com.android.tradefed.result.LogFileSaver; 50 import com.android.tradefed.result.TestSummary; 51 import com.android.tradefed.util.FileUtil; 52 import com.android.tradefed.util.StreamUtil; 53 import com.android.tradefed.util.TimeUtil; 54 import com.android.tradefed.util.ZipUtil; 55 56 import com.google.common.annotations.VisibleForTesting; 57 58 import org.xmlpull.v1.XmlPullParserException; 59 60 import java.io.File; 61 import java.io.FileInputStream; 62 import java.io.FileNotFoundException; 63 import java.io.IOException; 64 import java.io.InputStream; 65 import java.util.Arrays; 66 import java.util.Collections; 67 import java.util.HashSet; 68 import java.util.List; 69 import java.util.Map; 70 import java.util.Set; 71 import java.util.concurrent.CountDownLatch; 72 import java.util.concurrent.TimeUnit; 73 74 /** 75 * Collect test results for an entire invocation and output test results to disk. 76 */ 77 @OptionClass(alias="result-reporter") 78 public class ResultReporter implements ILogSaverListener, ITestInvocationListener, 79 ITestSummaryListener, IShardableListener { 80 81 private static final String UNKNOWN_DEVICE = "unknown_device"; 82 private static final String RESULT_KEY = "COMPATIBILITY_TEST_RESULT"; 83 private static final String CTS_PREFIX = "cts:"; 84 private static final String BUILD_INFO = CTS_PREFIX + "build_"; 85 86 private static final List<String> NOT_RETRY_FILES = Arrays.asList( 87 ChecksumReporter.NAME, 88 ChecksumReporter.PREV_NAME, 89 ResultHandler.FAILURE_REPORT_NAME); 90 91 @Option(name = CompatibilityTest.RETRY_OPTION, 92 shortName = 'r', 93 description = "retry a previous session.", 94 importance = Importance.IF_UNSET) 95 private Integer mRetrySessionId = null; 96 97 @Option(name = CompatibilityTest.RETRY_TYPE_OPTION, 98 description = "used with " + CompatibilityTest.RETRY_OPTION 99 + ", retry tests of a certain status. Possible values include \"failed\", " 100 + "\"not_executed\", and \"custom\".", 101 importance = Importance.IF_UNSET) 102 private RetryType mRetryType = null; 103 104 @Option(name = "result-server", description = "Server to publish test results.") 105 private String mResultServer; 106 107 @Option(name = "disable-result-posting", description = "Disable result posting into report server.") 108 private boolean mDisableResultPosting = false; 109 110 @Option(name = "include-test-log-tags", description = "Include test log tags in report.") 111 private boolean mIncludeTestLogTags = false; 112 113 @Option(name = "use-log-saver", description = "Also saves generated result with log saver") 114 private boolean mUseLogSaver = false; 115 116 @Option(name = "compress-logs", description = "Whether logs will be saved with compression") 117 private boolean mCompressLogs = true; 118 119 private CompatibilityBuildHelper mBuildHelper; 120 private File mResultDir = null; 121 private File mLogDir = null; 122 private ResultUploader mUploader; 123 private String mReferenceUrl; 124 private ILogSaver mLogSaver; 125 private int invocationEndedCount = 0; 126 private CountDownLatch mFinalized = null; 127 128 private IInvocationResult mResult = new InvocationResult(); 129 private IModuleResult mCurrentModuleResult; 130 private ICaseResult mCurrentCaseResult; 131 private ITestResult mCurrentResult; 132 private String mDeviceSerial = UNKNOWN_DEVICE; 133 private Set<String> mMasterDeviceSerials = new HashSet<>(); 134 private Set<IBuildInfo> mMasterBuildInfos = new HashSet<>(); 135 136 // mCurrentTestNum and mTotalTestsInModule track the progress within the module 137 // Note that this count is not necessarily equal to the count of tests contained 138 // in mCurrentModuleResult because of how special cases like ignored tests are reported. 139 private int mCurrentTestNum; 140 private int mTotalTestsInModule; 141 142 143 // Whether modules can be marked done for this invocation. Initialized in invocationStarted() 144 // Visible for unit testing 145 protected boolean mCanMarkDone; 146 // Whether the current module has previously been marked done 147 private boolean mModuleWasDone; 148 149 // Nullable. If null, "this" is considered the master and must handle 150 // result aggregation and reporting. When not null, it should forward events 151 // to the master. 152 private final ResultReporter mMasterResultReporter; 153 154 private LogFileSaver mTestLogSaver; 155 156 /** 157 * Default constructor. 158 */ 159 public ResultReporter() { 160 this(null); 161 mFinalized = new CountDownLatch(1); 162 } 163 164 /** 165 * Construct a shard ResultReporter that forwards module results to the 166 * masterResultReporter. 167 */ 168 public ResultReporter(ResultReporter masterResultReporter) { 169 mMasterResultReporter = masterResultReporter; 170 } 171 172 /** 173 * {@inheritDoc} 174 */ 175 @Override 176 public void invocationStarted(IInvocationContext context) { 177 IBuildInfo primaryBuild = context.getBuildInfos().get(0); 178 synchronized(this) { 179 if (mBuildHelper == null) { 180 mBuildHelper = new CompatibilityBuildHelper(primaryBuild); 181 } 182 if (mDeviceSerial == null && primaryBuild.getDeviceSerial() != null) { 183 mDeviceSerial = primaryBuild.getDeviceSerial(); 184 } 185 mCanMarkDone = canMarkDone(mBuildHelper.getRecentCommandLineArgs()); 186 } 187 188 if (isShardResultReporter()) { 189 // Shard ResultReporters forward invocationStarted to the mMasterResultReporter 190 mMasterResultReporter.invocationStarted(context); 191 return; 192 } 193 194 // NOTE: Everything after this line only applies to the master ResultReporter. 195 196 synchronized(this) { 197 if (primaryBuild.getDeviceSerial() != null) { 198 // The master ResultReporter collects all device serials being used 199 // for the current implementation. 200 mMasterDeviceSerials.add(primaryBuild.getDeviceSerial()); 201 } 202 203 // The master ResultReporter collects all buildInfos. 204 mMasterBuildInfos.add(primaryBuild); 205 206 if (mResultDir == null) { 207 // For the non-sharding case, invocationStarted is only called once, 208 // but for the sharding case, this might be called multiple times. 209 // Logic used to initialize the result directory should not be 210 // invoked twice during the same invocation. 211 initializeResultDirectories(); 212 } 213 } 214 } 215 216 /** 217 * Create directory structure where results and logs will be written. 218 */ 219 private void initializeResultDirectories() { 220 debug("Initializing result directory"); 221 222 try { 223 // Initialize the result directory. Either a new directory or reusing 224 // an existing session. 225 if (mRetrySessionId != null) { 226 // Overwrite the mResult with the test results of the previous session 227 mResult = ResultHandler.findResult(mBuildHelper.getResultsDir(), mRetrySessionId); 228 } 229 mResult.setStartTime(mBuildHelper.getStartTime()); 230 mResultDir = mBuildHelper.getResultDir(); 231 if (mResultDir != null) { 232 mResultDir.mkdirs(); 233 } 234 } catch (FileNotFoundException e) { 235 throw new RuntimeException(e); 236 } 237 238 if (mResultDir == null) { 239 throw new RuntimeException("Result Directory was not created"); 240 } 241 if (!mResultDir.exists()) { 242 throw new RuntimeException("Result Directory was not created: " + 243 mResultDir.getAbsolutePath()); 244 } 245 246 debug("Results Directory: " + mResultDir.getAbsolutePath()); 247 248 mUploader = new ResultUploader(mResultServer, mBuildHelper.getSuiteName()); 249 try { 250 mLogDir = new File(mBuildHelper.getLogsDir(), 251 CompatibilityBuildHelper.getDirSuffix(mBuildHelper.getStartTime())); 252 } catch (FileNotFoundException e) { 253 CLog.e(e); 254 } 255 if (mLogDir != null && mLogDir.mkdirs()) { 256 debug("Created log dir %s", mLogDir.getAbsolutePath()); 257 } 258 if (mLogDir == null || !mLogDir.exists()) { 259 throw new IllegalArgumentException(String.format("Could not create log dir %s", 260 mLogDir.getAbsolutePath())); 261 } 262 if (mTestLogSaver == null) { 263 mTestLogSaver = new LogFileSaver(mLogDir); 264 } 265 } 266 267 /** 268 * {@inheritDoc} 269 */ 270 @Override 271 public void testRunStarted(String id, int numTests) { 272 if (mCurrentModuleResult != null && mCurrentModuleResult.getId().equals(id) 273 && mCurrentModuleResult.isDone()) { 274 // Modules run with JarHostTest treat each test class as a separate module, 275 // resulting in additional unexpected test runs. 276 // This case exists only for N 277 mTotalTestsInModule += numTests; 278 } else { 279 // Handle non-JarHostTest case 280 mCurrentModuleResult = mResult.getOrCreateModule(id); 281 mModuleWasDone = mCurrentModuleResult.isDone(); 282 if (!mModuleWasDone) { 283 // we only want to update testRun variables if the IModuleResult is not yet done 284 // otherwise leave testRun variables alone so isDone evaluates to true. 285 if (mCurrentModuleResult.getExpectedTestRuns() == 0) { 286 mCurrentModuleResult.setExpectedTestRuns(TestRunHandler.getTestRuns( 287 mBuildHelper, mCurrentModuleResult.getId())); 288 } 289 mCurrentModuleResult.addTestRun(); 290 } 291 // Reset counters 292 mTotalTestsInModule = numTests; 293 mCurrentTestNum = 0; 294 } 295 mCurrentModuleResult.inProgress(true); 296 } 297 298 /** 299 * {@inheritDoc} 300 */ 301 @Override 302 public void testStarted(TestIdentifier test) { 303 mCurrentCaseResult = mCurrentModuleResult.getOrCreateResult(test.getClassName()); 304 mCurrentResult = mCurrentCaseResult.getOrCreateResult(test.getTestName().trim()); 305 if (mCurrentResult.isRetry()) { 306 mCurrentResult.reset(); // clear result status for this invocation 307 } 308 mCurrentTestNum++; 309 } 310 311 /** 312 * {@inheritDoc} 313 */ 314 @Override 315 public void testEnded(TestIdentifier test, Map<String, String> metrics) { 316 if (mCurrentResult.getResultStatus() == TestStatus.FAIL) { 317 // Test has previously failed. 318 return; 319 } 320 // device test can have performance results in test metrics 321 String perfResult = metrics.get(RESULT_KEY); 322 ReportLog report = null; 323 if (perfResult != null) { 324 try { 325 report = ReportLog.parse(perfResult); 326 } catch (XmlPullParserException | IOException e) { 327 e.printStackTrace(); 328 } 329 } else { 330 // host test should be checked into MetricsStore. 331 report = MetricsStore.removeResult(mBuildHelper.getBuildInfo(), 332 mCurrentModuleResult.getAbi(), test.toString()); 333 } 334 if (mCurrentResult.getResultStatus() == null) { 335 // Only claim that we passed when we're certain our result was 336 // not any other state. 337 mCurrentResult.passed(report); 338 } 339 } 340 341 /** 342 * {@inheritDoc} 343 */ 344 @Override 345 public void testIgnored(TestIdentifier test) { 346 mCurrentResult.skipped(); 347 } 348 349 /** 350 * {@inheritDoc} 351 */ 352 @Override 353 public void testFailed(TestIdentifier test, String trace) { 354 mCurrentResult.failed(trace); 355 } 356 357 /** 358 * {@inheritDoc} 359 */ 360 @Override 361 public void testAssumptionFailure(TestIdentifier test, String trace) { 362 mCurrentResult.skipped(); 363 } 364 365 /** 366 * {@inheritDoc} 367 */ 368 @Override 369 public void testRunStopped(long elapsedTime) { 370 // ignore 371 } 372 373 /** 374 * {@inheritDoc} 375 */ 376 @Override 377 public void testRunEnded(long elapsedTime, Map<String, String> metrics) { 378 mCurrentModuleResult.inProgress(false); 379 mCurrentModuleResult.addRuntime(elapsedTime); 380 if (!mModuleWasDone && mCanMarkDone) { 381 // Only mark module done if status of the invocation allows it (mCanMarkDone) and 382 // if module has not already been marked done. 383 mCurrentModuleResult.setDone(mCurrentTestNum >= mTotalTestsInModule); 384 } 385 if (isShardResultReporter()) { 386 // Forward module results to the master. 387 mMasterResultReporter.mergeModuleResult(mCurrentModuleResult); 388 mCurrentModuleResult.resetTestRuns(); 389 mCurrentModuleResult.resetRuntime(); 390 } 391 } 392 393 /** 394 * Directly add a module result. Note: this method is meant to be used by 395 * a shard ResultReporter. 396 */ 397 private void mergeModuleResult(IModuleResult moduleResult) { 398 // This merges the results in moduleResult to any existing results already 399 // contained in mResult. This is useful for retries and allows the final 400 // report from a retry to contain all test results. 401 synchronized(this) { 402 mResult.mergeModuleResult(moduleResult); 403 } 404 } 405 406 /** 407 * {@inheritDoc} 408 */ 409 @Override 410 public void testRunFailed(String errorMessage) { 411 // ignore 412 } 413 414 /** 415 * {@inheritDoc} 416 */ 417 @Override 418 public TestSummary getSummary() { 419 // ignore 420 return null; 421 } 422 423 /** 424 * {@inheritDoc} 425 */ 426 @Override 427 public void putSummary(List<TestSummary> summaries) { 428 // This is safe to be invoked on either the master or a shard ResultReporter, 429 // but the value added to the report will be that of the master ResultReporter. 430 if (summaries.size() > 0) { 431 mReferenceUrl = summaries.get(0).getSummary().getString(); 432 } 433 } 434 435 /** 436 * {@inheritDoc} 437 */ 438 @Override 439 public void invocationEnded(long elapsedTime) { 440 if (isShardResultReporter()) { 441 // Shard ResultReporters report 442 mMasterResultReporter.invocationEnded(elapsedTime); 443 return; 444 } 445 446 // NOTE: Everything after this line only applies to the master ResultReporter. 447 448 synchronized(this) { 449 // The master ResultReporter tracks the progress of all invocations across 450 // shard ResultReporters. Writing results should not proceed until all 451 // ResultReporters have completed. 452 if (++invocationEndedCount < mMasterBuildInfos.size()) { 453 return; 454 } 455 finalizeResults(elapsedTime); 456 mFinalized.countDown(); 457 } 458 } 459 460 private void finalizeResults(long elapsedTime) { 461 // Add all device serials into the result to be serialized 462 for (String deviceSerial : mMasterDeviceSerials) { 463 mResult.addDeviceSerial(deviceSerial); 464 } 465 466 Set<String> allExpectedModules = new HashSet<>(); 467 // Add all build info to the result to be serialized 468 for (IBuildInfo buildInfo : mMasterBuildInfos) { 469 for (Map.Entry<String, String> entry : buildInfo.getBuildAttributes().entrySet()) { 470 String key = entry.getKey(); 471 String value = entry.getValue(); 472 if (key.startsWith(BUILD_INFO)) { 473 mResult.addInvocationInfo(key.substring(CTS_PREFIX.length()), value); 474 } 475 476 if (key.equals(CompatibilityBuildHelper.MODULE_IDS) && value.length() > 0) { 477 Collections.addAll(allExpectedModules, value.split(",")); 478 } 479 } 480 } 481 482 // Include a record in the report of all expected modules ids, even if they weren't 483 // executed. 484 for (String moduleId : allExpectedModules) { 485 mResult.getOrCreateModule(moduleId); 486 } 487 488 String moduleProgress = String.format("%d of %d", 489 mResult.getModuleCompleteCount(), mResult.getModules().size()); 490 491 long startTime = mResult.getStartTime(); 492 try { 493 // Zip the full test results directory. 494 copyDynamicConfigFiles(mBuildHelper.getDynamicConfigFiles(), mResultDir); 495 copyFormattingFiles(mResultDir, mBuildHelper.getSuiteName()); 496 497 File resultFile = ResultHandler.writeResults(mBuildHelper.getSuiteName(), 498 mBuildHelper.getSuiteVersion(), mBuildHelper.getSuitePlan(), 499 mBuildHelper.getSuiteBuild(), mResult, mResultDir, startTime, 500 elapsedTime + startTime, mReferenceUrl, getLogUrl(), 501 mBuildHelper.getCommandLineArgs()); 502 if (mRetrySessionId != null) { 503 copyRetryFiles(ResultHandler.getResultDirectory( 504 mBuildHelper.getResultsDir(), mRetrySessionId), mResultDir); 505 } 506 File zippedResults = zipResults(mResultDir); 507 // Create failure report after zip file so extra data is not uploaded 508 File failureReport = ResultHandler.createFailureReport(resultFile); 509 if (failureReport.exists()) { 510 info("Test Result: %s", failureReport.getCanonicalPath()); 511 } else { 512 info("Test Result: %s", resultFile.getCanonicalPath()); 513 } 514 info("Test Logs: %s", mLogDir.getCanonicalPath()); 515 debug("Full Result: %s", zippedResults.getCanonicalPath()); 516 517 saveLog(resultFile, zippedResults); 518 519 uploadResult(resultFile); 520 521 } catch (IOException | XmlPullParserException e) { 522 CLog.e("[%s] Exception while saving result XML.", mDeviceSerial); 523 CLog.e(e); 524 } 525 // print the run results last. 526 info("Invocation finished in %s. PASSED: %d, FAILED: %d, MODULES: %s", 527 TimeUtil.formatElapsedTime(elapsedTime), 528 mResult.countResults(TestStatus.PASS), 529 mResult.countResults(TestStatus.FAIL), 530 moduleProgress); 531 } 532 533 /** 534 * {@inheritDoc} 535 */ 536 @Override 537 public void invocationFailed(Throwable cause) { 538 warn("Invocation failed: %s", cause); 539 InvocationFailureHandler.setFailed(mBuildHelper, cause); 540 } 541 542 /** 543 * {@inheritDoc} 544 */ 545 @Override 546 public void testLog(String name, LogDataType type, InputStreamSource stream) { 547 // This is safe to be invoked on either the master or a shard ResultReporter 548 if (isShardResultReporter()) { 549 // Shard ResultReporters forward testLog to the mMasterResultReporter 550 mMasterResultReporter.testLog(name, type, stream); 551 return; 552 } 553 try { 554 File logFile = null; 555 if (mCompressLogs) { 556 logFile = mTestLogSaver.saveAndGZipLogData(name, type, stream.createInputStream()); 557 } else { 558 logFile = mTestLogSaver.saveLogData(name, type, stream.createInputStream()); 559 } 560 debug("Saved logs for %s in %s", name, logFile.getAbsolutePath()); 561 } catch (IOException e) { 562 warn("Failed to write log for %s", name); 563 CLog.e(e); 564 } 565 } 566 567 /** 568 * {@inheritDoc} 569 */ 570 @Override 571 public void testLogSaved(String dataName, LogDataType dataType, InputStreamSource dataStream, 572 LogFile logFile) { 573 // This is safe to be invoked on either the master or a shard ResultReporter 574 if (mIncludeTestLogTags && mCurrentResult != null 575 && dataName.startsWith(mCurrentResult.getFullName())) { 576 577 if (dataType == LogDataType.BUGREPORT) { 578 mCurrentResult.setBugReport(logFile.getUrl()); 579 } else if (dataType == LogDataType.LOGCAT) { 580 mCurrentResult.setLog(logFile.getUrl()); 581 } else if (dataType == LogDataType.PNG) { 582 mCurrentResult.setScreenshot(logFile.getUrl()); 583 } 584 } 585 } 586 587 /** 588 * {@inheritDoc} 589 */ 590 @Override 591 public void setLogSaver(ILogSaver saver) { 592 // This is safe to be invoked on either the master or a shard ResultReporter 593 mLogSaver = saver; 594 } 595 596 /** 597 * When enabled, save log data using log saver 598 */ 599 private void saveLog(File resultFile, File zippedResults) throws IOException { 600 if (!mUseLogSaver) { 601 return; 602 } 603 604 FileInputStream fis = null; 605 LogFile logFile = null; 606 try { 607 fis = new FileInputStream(resultFile); 608 logFile = mLogSaver.saveLogData("log-result", LogDataType.XML, fis); 609 debug("Result XML URL: %s", logFile.getUrl()); 610 } catch (IOException ioe) { 611 CLog.e("[%s] error saving XML with log saver", mDeviceSerial); 612 CLog.e(ioe); 613 } finally { 614 StreamUtil.close(fis); 615 } 616 // Save the full results folder. 617 if (zippedResults != null) { 618 FileInputStream zipResultStream = null; 619 try { 620 zipResultStream = new FileInputStream(zippedResults); 621 logFile = mLogSaver.saveLogData("results", LogDataType.ZIP, zipResultStream); 622 debug("Result zip URL: %s", logFile.getUrl()); 623 } finally { 624 StreamUtil.close(zipResultStream); 625 } 626 } 627 } 628 629 /** 630 * Return the path in which log saver persists log files or null if 631 * logSaver is not enabled. 632 */ 633 private String getLogUrl() { 634 if (!mUseLogSaver || mLogSaver == null) { 635 return null; 636 } 637 638 return mLogSaver.getLogReportDir().getUrl(); 639 } 640 641 @Override 642 public IShardableListener clone() { 643 ResultReporter clone = new ResultReporter(this); 644 OptionCopier.copyOptionsNoThrow(this, clone); 645 return clone; 646 } 647 648 /** 649 * Return true if this instance is a shard ResultReporter and should propagate 650 * certain events to the master. 651 */ 652 private boolean isShardResultReporter() { 653 return mMasterResultReporter != null; 654 } 655 656 /** 657 * When enabled, upload the result to a server. 658 */ 659 private void uploadResult(File resultFile) { 660 if (mResultServer != null && !mResultServer.trim().isEmpty() && !mDisableResultPosting) { 661 try { 662 debug("Result Server: %d", mUploader.uploadResult(resultFile, mReferenceUrl)); 663 } catch (IOException ioe) { 664 CLog.e("[%s] IOException while uploading result.", mDeviceSerial); 665 CLog.e(ioe); 666 } 667 } 668 } 669 670 /** 671 * Returns whether it is safe to mark modules as "done", given the invocation command-line 672 * arguments. Returns true unless this is a retry and specific filtering techniques are applied 673 * on the command-line, such as: 674 * --retry-type failed 675 * --include-filter 676 * --exclude-filter 677 * -t/--test 678 * --subplan 679 */ 680 private boolean canMarkDone(String args) { 681 if (mRetrySessionId == null) { 682 return true; // always allow modules to be marked done if not retry 683 } 684 return !(RetryType.FAILED.equals(mRetryType) 685 || RetryType.CUSTOM.equals(mRetryType) 686 || args.contains(CompatibilityTest.INCLUDE_FILTER_OPTION) 687 || args.contains(CompatibilityTest.EXCLUDE_FILTER_OPTION) 688 || args.contains(CompatibilityTest.SUBPLAN_OPTION) 689 || args.matches(String.format(".* (-%s|--%s) .*", 690 CompatibilityTest.TEST_OPTION_SHORT_NAME, CompatibilityTest.TEST_OPTION))); 691 } 692 693 /** 694 * Copy the xml formatting files stored in this jar to the results directory 695 * 696 * @param resultsDir 697 */ 698 static void copyFormattingFiles(File resultsDir, String suiteName) { 699 for (String resultFileName : ResultHandler.RESULT_RESOURCES) { 700 InputStream configStream = ResultHandler.class.getResourceAsStream( 701 String.format("/report/%s-%s", suiteName, resultFileName)); 702 if (configStream == null) { 703 // If suite specific files are not available, fallback to common. 704 configStream = ResultHandler.class.getResourceAsStream( 705 String.format("/report/%s", resultFileName)); 706 } 707 if (configStream != null) { 708 File resultFile = new File(resultsDir, resultFileName); 709 try { 710 FileUtil.writeToFile(configStream, resultFile); 711 } catch (IOException e) { 712 warn("Failed to write %s to file", resultFileName); 713 } 714 } else { 715 warn("Failed to load %s from jar", resultFileName); 716 } 717 } 718 } 719 720 /** 721 * move the dynamic config files to the results directory 722 * 723 * @param configFiles 724 * @param resultsDir 725 */ 726 static void copyDynamicConfigFiles(Map<String, File> configFiles, File resultsDir) { 727 if (configFiles.size() == 0) return; 728 729 File folder = new File(resultsDir, "config"); 730 folder.mkdir(); 731 for (String moduleName : configFiles.keySet()) { 732 File resultFile = new File(folder, moduleName+".dynamic"); 733 try { 734 FileUtil.copyFile(configFiles.get(moduleName), resultFile); 735 FileUtil.deleteFile(configFiles.get(moduleName)); 736 } catch (IOException e) { 737 warn("Failed to copy config file for %s to file", moduleName); 738 } 739 } 740 } 741 742 /** 743 * Recursively copy any other files found in the previous session's result directory to the 744 * new result directory, so long as they don't already exist. For example, a "screenshots" 745 * directory generated in a previous session by a passing test will not be generated on retry 746 * unless copied from the old result directory. 747 * 748 * @param oldDir 749 * @param newDir 750 */ 751 static void copyRetryFiles(File oldDir, File newDir) { 752 File[] oldChildren = oldDir.listFiles(); 753 for (File oldChild : oldChildren) { 754 if (NOT_RETRY_FILES.contains(oldChild.getName())) { 755 continue; // do not copy this file/directory or its children 756 } 757 File newChild = new File(newDir, oldChild.getName()); 758 if (!newChild.exists()) { 759 // If this old file or directory doesn't exist in new dir, simply copy it 760 try { 761 if (oldChild.isDirectory()) { 762 FileUtil.recursiveCopy(oldChild, newChild); 763 } else { 764 FileUtil.copyFile(oldChild, newChild); 765 } 766 } catch (IOException e) { 767 warn("Failed to copy file \"%s\" from previous session", oldChild.getName()); 768 } 769 } else if (oldChild.isDirectory() && newChild.isDirectory()) { 770 // If both children exist as directories, make sure the children of the old child 771 // directory exist in the new child directory. 772 copyRetryFiles(oldChild, newChild); 773 } 774 } 775 } 776 777 /** 778 * Zip the contents of the given results directory. 779 * 780 * @param resultsDir 781 */ 782 private static File zipResults(File resultsDir) { 783 File zipResultFile = null; 784 try { 785 // create a file in parent directory, with same name as resultsDir 786 zipResultFile = new File(resultsDir.getParent(), String.format("%s.zip", 787 resultsDir.getName())); 788 ZipUtil.createZip(resultsDir, zipResultFile); 789 } catch (IOException e) { 790 warn("Failed to create zip for %s", resultsDir.getName()); 791 } 792 return zipResultFile; 793 } 794 795 /** 796 * Log info to the console. 797 */ 798 private static void info(String format, Object... args) { 799 log(LogLevel.INFO, format, args); 800 } 801 802 /** 803 * Log debug to the console. 804 */ 805 private static void debug(String format, Object... args) { 806 log(LogLevel.DEBUG, format, args); 807 } 808 809 /** 810 * Log a warning to the console. 811 */ 812 private static void warn(String format, Object... args) { 813 log(LogLevel.WARN, format, args); 814 } 815 816 /** 817 * Log a message to the console 818 */ 819 private static void log(LogLevel level, String format, Object... args) { 820 CLog.logAndDisplay(level, format, args); 821 } 822 823 /** 824 * For testing purpose. 825 */ 826 @VisibleForTesting 827 public IInvocationResult getResult() { 828 return mResult; 829 } 830 831 /** 832 * Returns true if the reporter is finalized before the end of the timeout. False otherwise. 833 */ 834 @VisibleForTesting 835 public boolean waitForFinalized(long timeout, TimeUnit unit) throws InterruptedException { 836 return mFinalized.await(timeout, unit); 837 } 838 } 839