1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.suite; 17 18 import com.android.ddmlib.Log.LogLevel; 19 import com.android.tradefed.build.IBuildInfo; 20 import com.android.tradefed.config.ConfigurationException; 21 import com.android.tradefed.config.IConfiguration; 22 import com.android.tradefed.config.Option; 23 import com.android.tradefed.config.OptionCopier; 24 import com.android.tradefed.device.DeviceNotAvailableException; 25 import com.android.tradefed.device.ITestDevice; 26 import com.android.tradefed.log.LogUtil.CLog; 27 import com.android.tradefed.result.ITestInvocationListener; 28 import com.android.tradefed.result.InputStreamSource; 29 import com.android.tradefed.result.LogDataType; 30 import com.android.tradefed.suite.checker.ISystemStatusChecker; 31 import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver; 32 import com.android.tradefed.testtype.IBuildReceiver; 33 import com.android.tradefed.testtype.IDeviceTest; 34 import com.android.tradefed.testtype.IRemoteTest; 35 import com.android.tradefed.testtype.IShardableTest; 36 import com.android.tradefed.testtype.ITestCollector; 37 38 import java.util.ArrayList; 39 import java.util.Collection; 40 import java.util.Collections; 41 import java.util.LinkedHashMap; 42 import java.util.List; 43 import java.util.Map.Entry; 44 45 /** 46 * Abstract class used to run Test Suite. This class provide the base of how the Suite will be run. 47 * Each implementation can define the list of tests via the {@link #loadTests()} method. 48 */ 49 public abstract class ITestSuite 50 implements IRemoteTest, 51 IDeviceTest, 52 IBuildReceiver, 53 ISystemStatusCheckerReceiver, 54 IShardableTest, 55 ITestCollector { 56 57 public static final String MODULE_CHECKER_PRE = "PreModuleChecker"; 58 public static final String MODULE_CHECKER_POST = "PostModuleChecker"; 59 60 // Options for test failure case 61 @Option( 62 name = "bugreport-on-failure", 63 description = 64 "Take a bugreport on every test failure. Warning: This may require a lot" 65 + "of storage space of the machine running the tests." 66 ) 67 private boolean mBugReportOnFailure = false; 68 69 @Option(name = "logcat-on-failure", 70 description = "Take a logcat snapshot on every test failure.") 71 private boolean mLogcatOnFailure = false; 72 73 @Option(name = "logcat-on-failure-size", 74 description = "The max number of logcat data in bytes to capture when " 75 + "--logcat-on-failure is on. Should be an amount that can comfortably fit in memory.") 76 private int mMaxLogcatBytes = 500 * 1024; // 500K 77 78 @Option(name = "screenshot-on-failure", 79 description = "Take a screenshot on every test failure.") 80 private boolean mScreenshotOnFailure = false; 81 82 @Option(name = "reboot-on-failure", 83 description = "Reboot the device after every test failure.") 84 private boolean mRebootOnFailure = false; 85 86 // Options for suite runner behavior 87 @Option(name = "reboot-per-module", description = "Reboot the device before every module run.") 88 private boolean mRebootPerModule = false; 89 90 @Option(name = "skip-all-system-status-check", 91 description = "Whether all system status check between modules should be skipped") 92 private boolean mSkipAllSystemStatusCheck = false; 93 94 @Option( 95 name = "report-system-checkers", 96 description = "Whether reporting system checkers as test or not." 97 ) 98 private boolean mReportSystemChecker = false; 99 100 @Option( 101 name = "collect-tests-only", 102 description = 103 "Only invoke the suite to collect list of applicable test cases. All " 104 + "test run callbacks will be triggered, but test execution will not be " 105 + "actually carried out." 106 ) 107 private boolean mCollectTestsOnly = false; 108 109 private ITestDevice mDevice; 110 private IBuildInfo mBuildInfo; 111 private List<ISystemStatusChecker> mSystemStatusCheckers; 112 113 // Sharding attributes 114 private boolean mIsSharded = false; 115 private ModuleDefinition mDirectModule = null; 116 private boolean mShouldMakeDynamicModule = true; 117 118 /** 119 * Abstract method to load the tests configuration that will be run. Each tests is defined by a 120 * {@link IConfiguration} and a unique name under which it will report results. 121 */ 122 public abstract LinkedHashMap<String, IConfiguration> loadTests(); 123 124 /** 125 * Return an instance of the class implementing {@link ITestSuite}. 126 */ 127 private ITestSuite createInstance() { 128 try { 129 return this.getClass().newInstance(); 130 } catch (InstantiationException | IllegalAccessException e) { 131 throw new RuntimeException(e); 132 } 133 } 134 135 /** Helper that creates and returns the list of {@link ModuleDefinition} to be executed. */ 136 private List<ModuleDefinition> createExecutionList() { 137 List<ModuleDefinition> runModules = new ArrayList<>(); 138 if (mDirectModule != null) { 139 // If we are sharded and already know what to run then we just do it. 140 runModules.add(mDirectModule); 141 mDirectModule.setDevice(mDevice); 142 mDirectModule.setBuild(mBuildInfo); 143 return runModules; 144 } 145 146 LinkedHashMap<String, IConfiguration> runConfig = loadTests(); 147 if (runConfig.isEmpty()) { 148 CLog.i("No config were loaded. Nothing to run."); 149 return runModules; 150 } 151 152 for (Entry<String, IConfiguration> config : runConfig.entrySet()) { 153 if (!ValidateSuiteConfigHelper.validateConfig(config.getValue())) { 154 throw new RuntimeException( 155 new ConfigurationException( 156 String.format( 157 "Configuration %s cannot be run in a suite.", 158 config.getValue().getName()))); 159 } 160 ModuleDefinition module = new ModuleDefinition(config.getKey(), 161 config.getValue().getTests(), config.getValue().getTargetPreparers()); 162 module.setDevice(mDevice); 163 module.setBuild(mBuildInfo); 164 runModules.add(module); 165 } 166 // Free the map once we are done with it. 167 runConfig = null; 168 return runModules; 169 } 170 171 /** Generic run method for all test loaded from {@link #loadTests()}. */ 172 @Override 173 public final void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 174 List<ModuleDefinition> runModules = createExecutionList(); 175 // Check if we have something to run. 176 if (runModules.isEmpty()) { 177 CLog.i("No tests to be run."); 178 return; 179 } 180 181 /** Setup a special listener to take actions on test failures. */ 182 TestFailureListener failureListener = 183 new TestFailureListener( 184 listener, 185 getDevice(), 186 mBugReportOnFailure, 187 mLogcatOnFailure, 188 mScreenshotOnFailure, 189 mRebootOnFailure, 190 mMaxLogcatBytes); 191 192 // Only print the running log if we are going to run something. 193 if (runModules.get(0).hasTests()) { 194 CLog.logAndDisplay( 195 LogLevel.INFO, 196 "%s running %s modules: %s", 197 mDevice.getSerialNumber(), 198 runModules.size(), 199 runModules); 200 } 201 202 /** Run all the module, make sure to reduce the list to release resources as we go. */ 203 try { 204 while (!runModules.isEmpty()) { 205 ModuleDefinition module = runModules.remove(0); 206 // Before running the module we ensure it has tests at this point or skip completely 207 // to avoid running SystemCheckers and preparation for nothing. 208 if (module.hasTests()) { 209 continue; 210 } 211 runSingleModule(module, listener, failureListener); 212 } 213 } catch (DeviceNotAvailableException e) { 214 CLog.e( 215 "A DeviceNotAvailableException occurred, following modules did not run: %s", 216 runModules); 217 for (ModuleDefinition module : runModules) { 218 listener.testRunStarted(module.getId(), 0); 219 listener.testRunFailed("Module did not run due to device not available."); 220 listener.testRunEnded(0, Collections.emptyMap()); 221 } 222 throw e; 223 } 224 } 225 226 /** 227 * Helper method that handle running a single module logic. 228 * 229 * @param module The {@link ModuleDefinition} to be ran. 230 * @param listener The {@link ITestInvocationListener} where to report results 231 * @param failureListener The {@link TestFailureListener} that collect infos on failures. 232 * @throws DeviceNotAvailableException 233 */ 234 private void runSingleModule( 235 ModuleDefinition module, 236 ITestInvocationListener listener, 237 TestFailureListener failureListener) 238 throws DeviceNotAvailableException { 239 if (mRebootPerModule) { 240 if ("user".equals(mDevice.getProperty("ro.build.type"))) { 241 CLog.e( 242 "reboot-per-module should only be used during development, " 243 + "this is a\" user\" build device"); 244 } else { 245 CLog.d("Rebooting device before starting next module"); 246 mDevice.reboot(); 247 } 248 } 249 250 if (!mSkipAllSystemStatusCheck) { 251 runPreModuleCheck(module.getId(), mSystemStatusCheckers, mDevice, listener); 252 } 253 if (mCollectTestsOnly) { 254 module.setCollectTestsOnly(mCollectTestsOnly); 255 } 256 // Actually run the module 257 module.run(listener, failureListener); 258 259 if (!mSkipAllSystemStatusCheck) { 260 runPostModuleCheck(module.getId(), mSystemStatusCheckers, mDevice, listener); 261 } 262 } 263 264 /** 265 * Helper to run the System Status checkers preExecutionChecks defined for the test and log 266 * their failures. 267 */ 268 private void runPreModuleCheck( 269 String moduleName, 270 List<ISystemStatusChecker> checkers, 271 ITestDevice device, 272 ITestInvocationListener listener) 273 throws DeviceNotAvailableException { 274 long startTime = System.currentTimeMillis(); 275 CLog.i("Running system status checker before module execution: %s", moduleName); 276 List<String> failures = new ArrayList<>(); 277 for (ISystemStatusChecker checker : checkers) { 278 boolean result = checker.preExecutionCheck(device); 279 if (!result) { 280 failures.add(checker.getClass().getCanonicalName()); 281 CLog.w("System status checker [%s] failed", checker.getClass().getCanonicalName()); 282 } 283 } 284 if (!failures.isEmpty()) { 285 CLog.w("There are failed system status checkers: %s capturing a bugreport", 286 failures.toString()); 287 InputStreamSource bugSource = device.getBugreport(); 288 listener.testLog( 289 String.format("bugreport-checker-pre-module-%s", moduleName), 290 LogDataType.BUGREPORT, 291 bugSource); 292 bugSource.cancel(); 293 } 294 295 // We report System checkers like tests. 296 reportModuleCheckerResult(MODULE_CHECKER_PRE, moduleName, failures, startTime, listener); 297 } 298 299 /** 300 * Helper to run the System Status checkers postExecutionCheck defined for the test and log 301 * their failures. 302 */ 303 private void runPostModuleCheck( 304 String moduleName, 305 List<ISystemStatusChecker> checkers, 306 ITestDevice device, 307 ITestInvocationListener listener) 308 throws DeviceNotAvailableException { 309 long startTime = System.currentTimeMillis(); 310 CLog.i("Running system status checker after module execution: %s", moduleName); 311 List<String> failures = new ArrayList<>(); 312 for (ISystemStatusChecker checker : checkers) { 313 boolean result = checker.postExecutionCheck(device); 314 if (!result) { 315 failures.add(checker.getClass().getCanonicalName()); 316 CLog.w("System status checker [%s] failed", checker.getClass().getCanonicalName()); 317 } 318 } 319 if (!failures.isEmpty()) { 320 CLog.w("There are failed system status checkers: %s capturing a bugreport", 321 failures.toString()); 322 InputStreamSource bugSource = device.getBugreport(); 323 listener.testLog( 324 String.format("bugreport-checker-post-module-%s", moduleName), 325 LogDataType.BUGREPORT, 326 bugSource); 327 bugSource.cancel(); 328 } 329 330 // We report System checkers like tests. 331 reportModuleCheckerResult(MODULE_CHECKER_POST, moduleName, failures, startTime, listener); 332 } 333 334 /** Helper to report status checker results as test results. */ 335 private void reportModuleCheckerResult( 336 String identifier, 337 String moduleName, 338 List<String> failures, 339 long startTime, 340 ITestInvocationListener listener) { 341 if (!mReportSystemChecker) { 342 // do not log here, otherwise it could be very verbose. 343 return; 344 } 345 // Avoid messing with the final test count by making them empty runs. 346 listener.testRunStarted(identifier + "_" + moduleName, 0); 347 if (!failures.isEmpty()) { 348 listener.testRunFailed(String.format("%s failed '%s' checkers", moduleName, failures)); 349 } 350 listener.testRunEnded(System.currentTimeMillis() - startTime, Collections.emptyMap()); 351 } 352 353 /** {@inheritDoc} */ 354 @Override 355 public Collection<IRemoteTest> split(int shardCountHint) { 356 if (shardCountHint <= 1 || mIsSharded) { 357 // cannot shard or already sharded 358 return null; 359 } 360 361 LinkedHashMap<String, IConfiguration> runConfig = loadTests(); 362 if (runConfig.isEmpty()) { 363 CLog.i("No config were loaded. Nothing to run."); 364 return null; 365 } 366 injectInfo(runConfig); 367 368 // We split individual tests on double the shardCountHint to provide better average. 369 // The test pool mechanism prevent this from creating too much overhead. 370 List<ModuleDefinition> splitModules = 371 ModuleSplitter.splitConfiguration( 372 runConfig, shardCountHint, mShouldMakeDynamicModule); 373 runConfig.clear(); 374 runConfig = null; 375 // create an association of one ITestSuite <=> one ModuleDefinition as the smallest 376 // execution unit supported. 377 List<IRemoteTest> splitTests = new ArrayList<>(); 378 for (ModuleDefinition m : splitModules) { 379 ITestSuite suite = createInstance(); 380 OptionCopier.copyOptionsNoThrow(this, suite); 381 suite.mIsSharded = true; 382 suite.mDirectModule = m; 383 splitTests.add(suite); 384 } 385 // return the list of ITestSuite with their ModuleDefinition assigned 386 return splitTests; 387 } 388 389 /** 390 * Inject {@link ITestDevice} and {@link IBuildInfo} to the {@link IRemoteTest}s in the config 391 * before sharding since they may be needed. 392 */ 393 private void injectInfo(LinkedHashMap<String, IConfiguration> runConfig) { 394 for (IConfiguration config : runConfig.values()) { 395 for (IRemoteTest test : config.getTests()) { 396 if (test instanceof IBuildReceiver) { 397 ((IBuildReceiver) test).setBuild(mBuildInfo); 398 } 399 if (test instanceof IDeviceTest) { 400 ((IDeviceTest) test).setDevice(mDevice); 401 } 402 } 403 } 404 } 405 406 /** {@inheritDoc} */ 407 @Override 408 public void setDevice(ITestDevice device) { 409 mDevice = device; 410 } 411 412 /** 413 * {@inheritDoc} 414 */ 415 @Override 416 public ITestDevice getDevice() { 417 return mDevice; 418 } 419 420 /** 421 * {@inheritDoc} 422 */ 423 @Override 424 public void setBuild(IBuildInfo buildInfo) { 425 mBuildInfo = buildInfo; 426 } 427 428 /** 429 * Implementation of {@link ITestSuite} may require the build info to load the tests. 430 */ 431 public IBuildInfo getBuildInfo() { 432 return mBuildInfo; 433 } 434 435 /** 436 * {@inheritDoc} 437 */ 438 @Override 439 public void setSystemStatusChecker(List<ISystemStatusChecker> systemCheckers) { 440 mSystemStatusCheckers = systemCheckers; 441 } 442 443 /** 444 * Run the test suite in collector only mode, this requires all the sub-tests to implements this 445 * interface too. 446 */ 447 @Override 448 public void setCollectTestsOnly(boolean shouldCollectTest) { 449 mCollectTestsOnly = shouldCollectTest; 450 } 451 452 /** 453 * When doing distributed sharding, we cannot have ModuleDefinition that shares tests in a pool 454 * otherwise intra-module sharding will not work, so we allow to disable it. 455 */ 456 public void setShouldMakeDynamicModule(boolean dynamicModule) { 457 mShouldMakeDynamicModule = dynamicModule; 458 } 459 } 460