1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.suite; 17 18 import com.android.ddmlib.Log.LogLevel; 19 import com.android.tradefed.build.IBuildInfo; 20 import com.android.tradefed.config.ConfigurationException; 21 import com.android.tradefed.config.IConfiguration; 22 import com.android.tradefed.config.Option; 23 import com.android.tradefed.config.OptionCopier; 24 import com.android.tradefed.device.DeviceNotAvailableException; 25 import com.android.tradefed.device.ITestDevice; 26 import com.android.tradefed.invoker.IInvocationContext; 27 import com.android.tradefed.log.LogUtil.CLog; 28 import com.android.tradefed.result.ITestInvocationListener; 29 import com.android.tradefed.result.ITestLoggerReceiver; 30 import com.android.tradefed.result.InputStreamSource; 31 import com.android.tradefed.result.LogDataType; 32 import com.android.tradefed.suite.checker.ISystemStatusChecker; 33 import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver; 34 import com.android.tradefed.testtype.IBuildReceiver; 35 import com.android.tradefed.testtype.IDeviceTest; 36 import com.android.tradefed.testtype.IInvocationContextReceiver; 37 import com.android.tradefed.testtype.IMultiDeviceTest; 38 import com.android.tradefed.testtype.IRemoteTest; 39 import com.android.tradefed.testtype.IRuntimeHintProvider; 40 import com.android.tradefed.testtype.IShardableTest; 41 import com.android.tradefed.testtype.ITestCollector; 42 import com.android.tradefed.util.TimeUtil; 43 44 import java.util.ArrayList; 45 import java.util.Collection; 46 import java.util.Collections; 47 import java.util.LinkedHashMap; 48 import java.util.List; 49 import java.util.Map.Entry; 50 51 /** 52 * Abstract class used to run Test Suite. This class provide the base of how the Suite will be run. 53 * Each implementation can define the list of tests via the {@link #loadTests()} method. 54 */ 55 public abstract class ITestSuite 56 implements IRemoteTest, 57 IDeviceTest, 58 IBuildReceiver, 59 ISystemStatusCheckerReceiver, 60 IShardableTest, 61 ITestCollector, 62 IInvocationContextReceiver, 63 IRuntimeHintProvider { 64 65 public static final String MODULE_CHECKER_PRE = "PreModuleChecker"; 66 public static final String MODULE_CHECKER_POST = "PostModuleChecker"; 67 68 // Options for test failure case 69 @Option( 70 name = "bugreport-on-failure", 71 description = 72 "Take a bugreport on every test failure. Warning: This may require a lot" 73 + "of storage space of the machine running the tests." 74 ) 75 private boolean mBugReportOnFailure = false; 76 77 @Option(name = "logcat-on-failure", 78 description = "Take a logcat snapshot on every test failure.") 79 private boolean mLogcatOnFailure = false; 80 81 @Option(name = "logcat-on-failure-size", 82 description = "The max number of logcat data in bytes to capture when " 83 + "--logcat-on-failure is on. Should be an amount that can comfortably fit in memory.") 84 private int mMaxLogcatBytes = 500 * 1024; // 500K 85 86 @Option(name = "screenshot-on-failure", 87 description = "Take a screenshot on every test failure.") 88 private boolean mScreenshotOnFailure = false; 89 90 @Option(name = "reboot-on-failure", 91 description = "Reboot the device after every test failure.") 92 private boolean mRebootOnFailure = false; 93 94 // Options for suite runner behavior 95 @Option(name = "reboot-per-module", description = "Reboot the device before every module run.") 96 private boolean mRebootPerModule = false; 97 98 @Option(name = "skip-all-system-status-check", 99 description = "Whether all system status check between modules should be skipped") 100 private boolean mSkipAllSystemStatusCheck = false; 101 102 @Option( 103 name = "report-system-checkers", 104 description = "Whether reporting system checkers as test or not." 105 ) 106 private boolean mReportSystemChecker = false; 107 108 @Option( 109 name = "collect-tests-only", 110 description = 111 "Only invoke the suite to collect list of applicable test cases. All " 112 + "test run callbacks will be triggered, but test execution will not be " 113 + "actually carried out." 114 ) 115 private boolean mCollectTestsOnly = false; 116 117 private ITestDevice mDevice; 118 private IBuildInfo mBuildInfo; 119 private List<ISystemStatusChecker> mSystemStatusCheckers; 120 private IInvocationContext mContext; 121 122 // Sharding attributes 123 private boolean mIsSharded = false; 124 private ModuleDefinition mDirectModule = null; 125 private boolean mShouldMakeDynamicModule = true; 126 127 /** 128 * Abstract method to load the tests configuration that will be run. Each tests is defined by a 129 * {@link IConfiguration} and a unique name under which it will report results. 130 */ 131 public abstract LinkedHashMap<String, IConfiguration> loadTests(); 132 133 /** 134 * Return an instance of the class implementing {@link ITestSuite}. 135 */ 136 private ITestSuite createInstance() { 137 try { 138 return this.getClass().newInstance(); 139 } catch (InstantiationException | IllegalAccessException e) { 140 throw new RuntimeException(e); 141 } 142 } 143 144 /** Helper that creates and returns the list of {@link ModuleDefinition} to be executed. */ 145 private List<ModuleDefinition> createExecutionList() { 146 List<ModuleDefinition> runModules = new ArrayList<>(); 147 if (mDirectModule != null) { 148 // If we are sharded and already know what to run then we just do it. 149 runModules.add(mDirectModule); 150 mDirectModule.setDevice(mDevice); 151 mDirectModule.setDeviceInfos(mContext.getDeviceBuildMap()); 152 mDirectModule.setBuild(mBuildInfo); 153 return runModules; 154 } 155 156 LinkedHashMap<String, IConfiguration> runConfig = loadTests(); 157 if (runConfig.isEmpty()) { 158 CLog.i("No config were loaded. Nothing to run."); 159 return runModules; 160 } 161 162 for (Entry<String, IConfiguration> config : runConfig.entrySet()) { 163 if (!ValidateSuiteConfigHelper.validateConfig(config.getValue())) { 164 throw new RuntimeException( 165 new ConfigurationException( 166 String.format( 167 "Configuration %s cannot be run in a suite.", 168 config.getValue().getName()))); 169 } 170 ModuleDefinition module = 171 new ModuleDefinition( 172 config.getKey(), 173 config.getValue().getTests(), 174 config.getValue().getTargetPreparers(), 175 config.getValue().getMultiTargetPreparers(), 176 config.getValue().getConfigurationDescription()); 177 module.setDevice(mDevice); 178 module.setDeviceInfos(mContext.getDeviceBuildMap()); 179 module.setBuild(mBuildInfo); 180 runModules.add(module); 181 } 182 // Free the map once we are done with it. 183 runConfig = null; 184 return runModules; 185 } 186 187 /** Generic run method for all test loaded from {@link #loadTests()}. */ 188 @Override 189 public final void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 190 List<ModuleDefinition> runModules = createExecutionList(); 191 // Check if we have something to run. 192 if (runModules.isEmpty()) { 193 CLog.i("No tests to be run."); 194 return; 195 } 196 197 // Allow checkers to log files for easier debbuging. 198 for (ISystemStatusChecker checker : mSystemStatusCheckers) { 199 if (checker instanceof ITestLoggerReceiver) { 200 ((ITestLoggerReceiver) checker).setTestLogger(listener); 201 } 202 } 203 204 /** Setup a special listener to take actions on test failures. */ 205 TestFailureListener failureListener = 206 new TestFailureListener( 207 listener, 208 mContext.getDevices(), 209 mBugReportOnFailure, 210 mLogcatOnFailure, 211 mScreenshotOnFailure, 212 mRebootOnFailure, 213 mMaxLogcatBytes); 214 215 // Only print the running log if we are going to run something. 216 if (runModules.get(0).hasTests()) { 217 CLog.logAndDisplay( 218 LogLevel.INFO, 219 "%s running %s modules: %s", 220 mDevice.getSerialNumber(), 221 runModules.size(), 222 runModules); 223 } 224 225 /** Run all the module, make sure to reduce the list to release resources as we go. */ 226 try { 227 while (!runModules.isEmpty()) { 228 ModuleDefinition module = runModules.remove(0); 229 // Before running the module we ensure it has tests at this point or skip completely 230 // to avoid running SystemCheckers and preparation for nothing. 231 if (module.hasTests()) { 232 continue; 233 } 234 235 try { 236 listener.testModuleStarted(module.getModuleInvocationContext()); 237 // Populate the module context with devices and builds 238 for (String deviceName : mContext.getDeviceConfigNames()) { 239 module.getModuleInvocationContext() 240 .addAllocatedDevice(deviceName, mContext.getDevice(deviceName)); 241 module.getModuleInvocationContext() 242 .addDeviceBuildInfo(deviceName, mContext.getBuildInfo(deviceName)); 243 } 244 runSingleModule(module, listener, failureListener); 245 } finally { 246 // clear out module invocation context since we are now done with module 247 // execution 248 listener.testModuleEnded(); 249 } 250 } 251 } catch (DeviceNotAvailableException e) { 252 CLog.e( 253 "A DeviceNotAvailableException occurred, following modules did not run: %s", 254 runModules); 255 for (ModuleDefinition module : runModules) { 256 listener.testRunStarted(module.getId(), 0); 257 listener.testRunFailed("Module did not run due to device not available."); 258 listener.testRunEnded(0, Collections.emptyMap()); 259 } 260 throw e; 261 } 262 } 263 264 /** 265 * Helper method that handle running a single module logic. 266 * 267 * @param module The {@link ModuleDefinition} to be ran. 268 * @param listener The {@link ITestInvocationListener} where to report results 269 * @param failureListener The {@link TestFailureListener} that collect infos on failures. 270 * @throws DeviceNotAvailableException 271 */ 272 private void runSingleModule( 273 ModuleDefinition module, 274 ITestInvocationListener listener, 275 TestFailureListener failureListener) 276 throws DeviceNotAvailableException { 277 if (mRebootPerModule) { 278 if ("user".equals(mDevice.getProperty("ro.build.type"))) { 279 CLog.e( 280 "reboot-per-module should only be used during development, " 281 + "this is a\" user\" build device"); 282 } else { 283 CLog.d("Rebooting device before starting next module"); 284 mDevice.reboot(); 285 } 286 } 287 288 if (!mSkipAllSystemStatusCheck) { 289 runPreModuleCheck(module.getId(), mSystemStatusCheckers, mDevice, listener); 290 } 291 if (mCollectTestsOnly) { 292 module.setCollectTestsOnly(mCollectTestsOnly); 293 } 294 // Actually run the module 295 module.run(listener, failureListener); 296 297 if (!mSkipAllSystemStatusCheck) { 298 runPostModuleCheck(module.getId(), mSystemStatusCheckers, mDevice, listener); 299 } 300 } 301 302 /** 303 * Helper to run the System Status checkers preExecutionChecks defined for the test and log 304 * their failures. 305 */ 306 private void runPreModuleCheck( 307 String moduleName, 308 List<ISystemStatusChecker> checkers, 309 ITestDevice device, 310 ITestInvocationListener listener) 311 throws DeviceNotAvailableException { 312 long startTime = System.currentTimeMillis(); 313 CLog.i("Running system status checker before module execution: %s", moduleName); 314 List<String> failures = new ArrayList<>(); 315 for (ISystemStatusChecker checker : checkers) { 316 boolean result = checker.preExecutionCheck(device); 317 if (!result) { 318 failures.add(checker.getClass().getCanonicalName()); 319 CLog.w("System status checker [%s] failed", checker.getClass().getCanonicalName()); 320 } 321 } 322 if (!failures.isEmpty()) { 323 CLog.w("There are failed system status checkers: %s capturing a bugreport", 324 failures.toString()); 325 try (InputStreamSource bugSource = device.getBugreport()) { 326 listener.testLog( 327 String.format("bugreport-checker-pre-module-%s", moduleName), 328 LogDataType.BUGREPORT, 329 bugSource); 330 } 331 } 332 333 // We report System checkers like tests. 334 reportModuleCheckerResult(MODULE_CHECKER_PRE, moduleName, failures, startTime, listener); 335 } 336 337 /** 338 * Helper to run the System Status checkers postExecutionCheck defined for the test and log 339 * their failures. 340 */ 341 private void runPostModuleCheck( 342 String moduleName, 343 List<ISystemStatusChecker> checkers, 344 ITestDevice device, 345 ITestInvocationListener listener) 346 throws DeviceNotAvailableException { 347 long startTime = System.currentTimeMillis(); 348 CLog.i("Running system status checker after module execution: %s", moduleName); 349 List<String> failures = new ArrayList<>(); 350 for (ISystemStatusChecker checker : checkers) { 351 boolean result = checker.postExecutionCheck(device); 352 if (!result) { 353 failures.add(checker.getClass().getCanonicalName()); 354 CLog.w("System status checker [%s] failed", checker.getClass().getCanonicalName()); 355 } 356 } 357 if (!failures.isEmpty()) { 358 CLog.w("There are failed system status checkers: %s capturing a bugreport", 359 failures.toString()); 360 try (InputStreamSource bugSource = device.getBugreport()) { 361 listener.testLog( 362 String.format("bugreport-checker-post-module-%s", moduleName), 363 LogDataType.BUGREPORT, 364 bugSource); 365 } 366 } 367 368 // We report System checkers like tests. 369 reportModuleCheckerResult(MODULE_CHECKER_POST, moduleName, failures, startTime, listener); 370 } 371 372 /** Helper to report status checker results as test results. */ 373 private void reportModuleCheckerResult( 374 String identifier, 375 String moduleName, 376 List<String> failures, 377 long startTime, 378 ITestInvocationListener listener) { 379 if (!mReportSystemChecker) { 380 // do not log here, otherwise it could be very verbose. 381 return; 382 } 383 // Avoid messing with the final test count by making them empty runs. 384 listener.testRunStarted(identifier + "_" + moduleName, 0); 385 if (!failures.isEmpty()) { 386 listener.testRunFailed(String.format("%s failed '%s' checkers", moduleName, failures)); 387 } 388 listener.testRunEnded(System.currentTimeMillis() - startTime, Collections.emptyMap()); 389 } 390 391 /** {@inheritDoc} */ 392 @Override 393 public Collection<IRemoteTest> split(int shardCountHint) { 394 if (shardCountHint <= 1 || mIsSharded) { 395 // cannot shard or already sharded 396 return null; 397 } 398 399 LinkedHashMap<String, IConfiguration> runConfig = loadTests(); 400 if (runConfig.isEmpty()) { 401 CLog.i("No config were loaded. Nothing to run."); 402 return null; 403 } 404 injectInfo(runConfig); 405 406 // We split individual tests on double the shardCountHint to provide better average. 407 // The test pool mechanism prevent this from creating too much overhead. 408 List<ModuleDefinition> splitModules = 409 ModuleSplitter.splitConfiguration( 410 runConfig, shardCountHint, mShouldMakeDynamicModule); 411 runConfig.clear(); 412 runConfig = null; 413 // create an association of one ITestSuite <=> one ModuleDefinition as the smallest 414 // execution unit supported. 415 List<IRemoteTest> splitTests = new ArrayList<>(); 416 for (ModuleDefinition m : splitModules) { 417 ITestSuite suite = createInstance(); 418 OptionCopier.copyOptionsNoThrow(this, suite); 419 suite.mIsSharded = true; 420 suite.mDirectModule = m; 421 splitTests.add(suite); 422 } 423 // return the list of ITestSuite with their ModuleDefinition assigned 424 return splitTests; 425 } 426 427 /** 428 * Inject {@link ITestDevice} and {@link IBuildInfo} to the {@link IRemoteTest}s in the config 429 * before sharding since they may be needed. 430 */ 431 private void injectInfo(LinkedHashMap<String, IConfiguration> runConfig) { 432 for (IConfiguration config : runConfig.values()) { 433 for (IRemoteTest test : config.getTests()) { 434 if (test instanceof IBuildReceiver) { 435 ((IBuildReceiver) test).setBuild(mBuildInfo); 436 } 437 if (test instanceof IDeviceTest) { 438 ((IDeviceTest) test).setDevice(mDevice); 439 } 440 if (test instanceof IMultiDeviceTest) { 441 ((IMultiDeviceTest) test).setDeviceInfos(mContext.getDeviceBuildMap()); 442 } 443 } 444 } 445 } 446 447 /** {@inheritDoc} */ 448 @Override 449 public void setDevice(ITestDevice device) { 450 mDevice = device; 451 } 452 453 /** 454 * {@inheritDoc} 455 */ 456 @Override 457 public ITestDevice getDevice() { 458 return mDevice; 459 } 460 461 /** 462 * {@inheritDoc} 463 */ 464 @Override 465 public void setBuild(IBuildInfo buildInfo) { 466 mBuildInfo = buildInfo; 467 } 468 469 /** 470 * Implementation of {@link ITestSuite} may require the build info to load the tests. 471 */ 472 public IBuildInfo getBuildInfo() { 473 return mBuildInfo; 474 } 475 476 /** 477 * {@inheritDoc} 478 */ 479 @Override 480 public void setSystemStatusChecker(List<ISystemStatusChecker> systemCheckers) { 481 mSystemStatusCheckers = systemCheckers; 482 } 483 484 /** 485 * Run the test suite in collector only mode, this requires all the sub-tests to implements this 486 * interface too. 487 */ 488 @Override 489 public void setCollectTestsOnly(boolean shouldCollectTest) { 490 mCollectTestsOnly = shouldCollectTest; 491 } 492 493 /** 494 * When doing distributed sharding, we cannot have ModuleDefinition that shares tests in a pool 495 * otherwise intra-module sharding will not work, so we allow to disable it. 496 */ 497 public void setShouldMakeDynamicModule(boolean dynamicModule) { 498 mShouldMakeDynamicModule = dynamicModule; 499 } 500 501 /** {@inheritDoc} */ 502 @Override 503 public void setInvocationContext(IInvocationContext invocationContext) { 504 mContext = invocationContext; 505 } 506 507 /** {@inheritDoc} */ 508 @Override 509 public long getRuntimeHint() { 510 if (mDirectModule != null) { 511 CLog.d( 512 " %s: %s", 513 mDirectModule.getId(), 514 TimeUtil.formatElapsedTime(mDirectModule.getRuntimeHint())); 515 return mDirectModule.getRuntimeHint(); 516 } 517 return 0l; 518 } 519 520 /** 521 * Returns the {@link ModuleDefinition} to be executed directly, or null if none yet (when the 522 * ITestSuite has not been sharded yet). 523 */ 524 public ModuleDefinition getDirectModule() { 525 return mDirectModule; 526 } 527 } 528