1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.cts.tradefed.testtype; 18 19 import com.android.cts.tradefed.build.CtsBuildHelper; 20 import com.android.cts.tradefed.device.DeviceInfoCollector; 21 import com.android.cts.tradefed.result.CtsTestStatus; 22 import com.android.cts.tradefed.result.PlanCreator; 23 import com.android.ddmlib.Log; 24 import com.android.ddmlib.Log.LogLevel; 25 import com.android.ddmlib.testrunner.TestIdentifier; 26 import com.android.tradefed.build.IBuildInfo; 27 import com.android.tradefed.config.ConfigurationException; 28 import com.android.tradefed.config.Option; 29 import com.android.tradefed.config.Option.Importance; 30 import com.android.tradefed.device.DeviceNotAvailableException; 31 import com.android.tradefed.device.ITestDevice; 32 import com.android.tradefed.log.LogUtil.CLog; 33 import com.android.tradefed.result.ITestInvocationListener; 34 import com.android.tradefed.result.InputStreamSource; 35 import com.android.tradefed.result.LogDataType; 36 import com.android.tradefed.result.ResultForwarder; 37 import com.android.tradefed.testtype.IBuildReceiver; 38 import com.android.tradefed.testtype.IDeviceTest; 39 import com.android.tradefed.testtype.IRemoteTest; 40 import com.android.tradefed.testtype.IResumableTest; 41 import com.android.tradefed.testtype.IShardableTest; 42 import com.android.tradefed.util.xml.AbstractXmlParser.ParseException; 43 44 import junit.framework.Test; 45 46 import java.io.BufferedInputStream; 47 import java.io.File; 48 import java.io.FileInputStream; 49 import java.io.FileNotFoundException; 50 import java.io.InputStream; 51 import java.util.ArrayList; 52 import java.util.Collection; 53 import java.util.HashMap; 54 import java.util.HashSet; 55 import java.util.LinkedHashSet; 56 import java.util.LinkedList; 57 import java.util.List; 58 import java.util.Map; 59 import java.util.Queue; 60 import java.util.Set; 61 62 /** 63 * A {@link Test} for running CTS tests. 64 * <p/> 65 * Supports running all the tests contained in a CTS plan, or individual test packages. 66 */ 67 public class CtsTest implements IDeviceTest, IResumableTest, IShardableTest, IBuildReceiver { 68 private static final String LOG_TAG = "CtsTest"; 69 70 public static final String PLAN_OPTION = "plan"; 71 private static final String PACKAGE_OPTION = "package"; 72 private static final String CLASS_OPTION = "class"; 73 private static final String METHOD_OPTION = "method"; 74 public static final String CONTINUE_OPTION = "continue-session"; 75 76 public static final String PACKAGE_NAME_METRIC = "packageName"; 77 public static final String PACKAGE_DIGEST_METRIC = "packageDigest"; 78 79 private ITestDevice mDevice; 80 81 @Option(name = PLAN_OPTION, description = "the test plan to run.", 82 importance = Importance.IF_UNSET) 83 private String mPlanName = null; 84 85 @Option(name = PACKAGE_OPTION, shortName = 'p', description = "the test packages(s) to run.", 86 importance = Importance.IF_UNSET) 87 private Collection<String> mPackageNames = new ArrayList<String>(); 88 89 @Option(name = "exclude-package", description = "the test packages(s) to exclude from the run.") 90 private Collection<String> mExcludedPackageNames = new ArrayList<String>(); 91 92 @Option(name = CLASS_OPTION, shortName = 'c', description = "run a specific test class.", 93 importance = Importance.IF_UNSET) 94 private String mClassName = null; 95 96 @Option(name = METHOD_OPTION, shortName = 'm', 97 description = "run a specific test method, from given --class.", 98 importance = Importance.IF_UNSET) 99 private String mMethodName = null; 100 101 @Option(name = CONTINUE_OPTION, 102 description = "continue a previous test session.", 103 importance = Importance.IF_UNSET) 104 private Integer mContinueSessionId = null; 105 106 @Option(name = "skip-device-info", shortName = 'd', description = 107 "flag to control whether to collect info from device. Providing this flag will speed up " + 108 "test execution for short test runs but will result in required data being omitted from " + 109 "the test report.") 110 private boolean mSkipDeviceInfo = false; 111 112 @Option(name = "resume", description = 113 "flag to attempt to automatically resume aborted test run on another connected device. ") 114 private boolean mResume = false; 115 116 @Option(name = "shards", description = 117 "shard the tests to run into separately runnable chunks to execute on multiple devices " + 118 "concurrently.") 119 private int mShards = 1; 120 121 @Option(name = "screenshot", description = 122 "flag for taking a screenshot of the device when test execution is complete.") 123 private boolean mScreenshot = false; 124 125 @Option(name = "bugreport", shortName = 'b', description = 126 "take a bugreport after each failed test. " + 127 "Warning: can potentially use a lot of disk space.") 128 private boolean mBugreport = false; 129 130 /** data structure for a {@link IRemoteTest} and its known tests */ 131 class TestPackage { 132 private final IRemoteTest mTestForPackage; 133 private final Collection<TestIdentifier> mKnownTests; 134 private final ITestPackageDef mPackageDef; 135 136 TestPackage(ITestPackageDef packageDef, IRemoteTest testForPackage, 137 Collection<TestIdentifier> knownTests) { 138 mPackageDef = packageDef; 139 mTestForPackage = testForPackage; 140 mKnownTests = knownTests; 141 } 142 143 IRemoteTest getTestForPackage() { 144 return mTestForPackage; 145 } 146 147 Collection<TestIdentifier> getKnownTests() { 148 return mKnownTests; 149 } 150 151 ITestPackageDef getPackageDef() { 152 return mPackageDef; 153 } 154 155 /** 156 * Return the test run name that should be used for the TestPackage 157 * @return 158 */ 159 String getTestRunName() { 160 return mPackageDef.getUri(); 161 } 162 } 163 164 /** 165 * A {@link ResultForwarder} that will forward a bugreport on each failed test. 166 */ 167 private static class FailedTestBugreportGenerator extends ResultForwarder { 168 private ITestDevice mDevice; 169 170 public FailedTestBugreportGenerator(ITestInvocationListener listener, ITestDevice device) { 171 super(listener); 172 mDevice = device; 173 } 174 175 @Override 176 public void testFailed(TestFailure status, TestIdentifier test, String trace) { 177 super.testFailed(status, test, trace); 178 InputStreamSource bugSource = mDevice.getBugreport(); 179 super.testLog(String.format("bug-%s", test.toString()), LogDataType.TEXT, 180 bugSource); 181 bugSource.cancel(); 182 } 183 } 184 185 /** list of remaining tests to execute */ 186 private List<TestPackage> mRemainingTestPkgs = null; 187 188 private CtsBuildHelper mCtsBuild = null; 189 private IBuildInfo mBuildInfo = null; 190 191 /** 192 * {@inheritDoc} 193 */ 194 public ITestDevice getDevice() { 195 return mDevice; 196 } 197 198 /** 199 * {@inheritDoc} 200 */ 201 public void setDevice(ITestDevice device) { 202 mDevice = device; 203 } 204 205 /** 206 * Set the plan name to run. 207 * <p/> 208 * Exposed for unit testing 209 */ 210 void setPlanName(String planName) { 211 mPlanName = planName; 212 } 213 214 /** 215 * Set the skip collect device info flag. 216 * <p/> 217 * Exposed for unit testing 218 */ 219 void setSkipDeviceInfo(boolean skipDeviceInfo) { 220 mSkipDeviceInfo = skipDeviceInfo; 221 } 222 223 /** 224 * Adds a package name to the list of test packages to run. 225 * <p/> 226 * Exposed for unit testing 227 */ 228 void addPackageName(String packageName) { 229 mPackageNames.add(packageName); 230 } 231 232 /** 233 * Adds a package name to the list of test packages to exclude. 234 * <p/> 235 * Exposed for unit testing 236 */ 237 void addExcludedPackageName(String packageName) { 238 mExcludedPackageNames.add(packageName); 239 } 240 241 /** 242 * Set the test class name to run. 243 * <p/> 244 * Exposed for unit testing 245 */ 246 void setClassName(String className) { 247 mClassName = className; 248 } 249 250 /** 251 * Set the test method name to run. 252 * <p/> 253 * Exposed for unit testing 254 */ 255 void setMethodName(String methodName) { 256 mMethodName = methodName; 257 } 258 259 /** 260 * Sets the test session id to continue. 261 * <p/> 262 * Exposed for unit testing 263 */ 264 void setContinueSessionId(int sessionId) { 265 mContinueSessionId = sessionId; 266 } 267 268 /** 269 * {@inheritDoc} 270 */ 271 @Override 272 public boolean isResumable() { 273 return mResume; 274 } 275 276 /** 277 * {@inheritDoc} 278 */ 279 @Override 280 public void setBuild(IBuildInfo build) { 281 mCtsBuild = CtsBuildHelper.createBuildHelper(build); 282 mBuildInfo = build; 283 } 284 285 /** 286 * Set the CTS build container. 287 * <p/> 288 * Exposed so unit tests can mock the provided build. 289 * 290 * @param buildHelper 291 */ 292 void setBuildHelper(CtsBuildHelper buildHelper) { 293 mCtsBuild = buildHelper; 294 } 295 296 /** 297 * {@inheritDoc} 298 */ 299 @Override 300 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 301 if (getDevice() == null) { 302 throw new IllegalArgumentException("missing device"); 303 } 304 305 if (mRemainingTestPkgs == null) { 306 checkFields(); 307 mRemainingTestPkgs = buildTestsToRun(); 308 } 309 if (mBugreport) { 310 FailedTestBugreportGenerator bugListener = new FailedTestBugreportGenerator(listener, 311 getDevice()); 312 listener = bugListener; 313 } 314 315 // collect and install the prerequisiteApks first, to save time when multiple test 316 // packages are using the same prerequisite apk (I'm looking at you, CtsTestStubs!) 317 Collection<String> prerequisiteApks = getPrerequisiteApks(mRemainingTestPkgs); 318 Collection<String> uninstallPackages = getPrerequisitePackageNames(mRemainingTestPkgs); 319 ResultFilter filter = new ResultFilter(listener, mRemainingTestPkgs); 320 321 try { 322 installPrerequisiteApks(prerequisiteApks); 323 324 // always collect the device info, even for resumed runs, since test will likely be 325 // running on a different device 326 collectDeviceInfo(getDevice(), mCtsBuild, listener); 327 328 while (!mRemainingTestPkgs.isEmpty()) { 329 TestPackage knownTests = mRemainingTestPkgs.get(0); 330 331 IRemoteTest test = knownTests.getTestForPackage(); 332 if (test instanceof IDeviceTest) { 333 ((IDeviceTest)test).setDevice(getDevice()); 334 } 335 if (test instanceof IBuildReceiver) { 336 ((IBuildReceiver)test).setBuild(mBuildInfo); 337 } 338 339 forwardPackageDetails(knownTests.getPackageDef(), listener); 340 test.run(filter); 341 mRemainingTestPkgs.remove(0); 342 } 343 344 if (mScreenshot) { 345 InputStreamSource screenshotSource = getDevice().getScreenshot(); 346 try { 347 listener.testLog("screenshot", LogDataType.PNG, screenshotSource); 348 } finally { 349 screenshotSource.cancel(); 350 } 351 } 352 353 uninstallPrequisiteApks(uninstallPackages); 354 355 } finally { 356 filter.reportUnexecutedTests(); 357 } 358 } 359 360 /** 361 * Build the list of test packages to run 362 * 363 * @return 364 */ 365 private List<TestPackage> buildTestsToRun() { 366 List<TestPackage> testPkgList = new LinkedList<TestPackage>(); 367 try { 368 ITestPackageRepo testRepo = createTestCaseRepo(); 369 Collection<ITestPackageDef> testPkgDefs = getTestPackagesToRun(testRepo); 370 371 for (ITestPackageDef testPkgDef : testPkgDefs) { 372 addTestPackage(testPkgList, testPkgDef); 373 } 374 if (testPkgList.isEmpty()) { 375 Log.logAndDisplay(LogLevel.WARN, LOG_TAG, "No tests to run"); 376 } 377 } catch (FileNotFoundException e) { 378 throw new IllegalArgumentException("failed to find CTS plan file", e); 379 } catch (ParseException e) { 380 throw new IllegalArgumentException("failed to parse CTS plan file", e); 381 } catch (ConfigurationException e) { 382 throw new IllegalArgumentException("failed to process arguments", e); 383 } 384 return testPkgList; 385 } 386 387 /** 388 * Adds a test package to the list of packages to test 389 * 390 * @param testList 391 * @param testPkgDef 392 */ 393 private void addTestPackage(List<TestPackage> testList, ITestPackageDef testPkgDef) { 394 IRemoteTest testForPackage = testPkgDef.createTest(mCtsBuild.getTestCasesDir()); 395 if (testForPackage != null) { 396 Collection<TestIdentifier> knownTests = testPkgDef.getTests(); 397 testList.add(new TestPackage(testPkgDef, testForPackage, knownTests)); 398 } 399 } 400 401 /** 402 * Return the list of test package defs to run 403 * 404 * @return the list of test package defs to run 405 * @throws ParseException 406 * @throws FileNotFoundException 407 * @throws ConfigurationException 408 */ 409 private Collection<ITestPackageDef> getTestPackagesToRun(ITestPackageRepo testRepo) 410 throws ParseException, FileNotFoundException, ConfigurationException { 411 // use LinkedHashSet to have predictable iteration order 412 Set<ITestPackageDef> testPkgDefs = new LinkedHashSet<ITestPackageDef>(); 413 if (mPlanName != null) { 414 Log.i(LOG_TAG, String.format("Executing CTS test plan %s", mPlanName)); 415 File ctsPlanFile = mCtsBuild.getTestPlanFile(mPlanName); 416 ITestPlan plan = createPlan(mPlanName); 417 plan.parse(createXmlStream(ctsPlanFile)); 418 for (String uri : plan.getTestUris()) { 419 if (!mExcludedPackageNames.contains(uri)) { 420 ITestPackageDef testPackage = testRepo.getTestPackage(uri); 421 testPackage.setExcludedTestFilter(plan.getExcludedTestFilter(uri)); 422 testPkgDefs.add(testPackage); 423 } 424 } 425 } else if (mPackageNames.size() > 0){ 426 Log.i(LOG_TAG, String.format("Executing CTS test packages %s", mPackageNames)); 427 for (String uri : mPackageNames) { 428 ITestPackageDef testPackage = testRepo.getTestPackage(uri); 429 if (testPackage != null) { 430 testPkgDefs.add(testPackage); 431 } else { 432 throw new IllegalArgumentException(String.format( 433 "Could not find test package %s. " + 434 "Use 'list packages' to see available packages." , uri)); 435 } 436 } 437 } else if (mClassName != null) { 438 Log.i(LOG_TAG, String.format("Executing CTS test class %s", mClassName)); 439 // try to find package to run from class name 440 String packageUri = testRepo.findPackageForTest(mClassName); 441 if (packageUri != null) { 442 ITestPackageDef testPackageDef = testRepo.getTestPackage(packageUri); 443 testPackageDef.setClassName(mClassName, mMethodName); 444 testPkgDefs.add(testPackageDef); 445 } else { 446 Log.logAndDisplay(LogLevel.WARN, LOG_TAG, String.format( 447 "Could not find package for test class %s", mClassName)); 448 } 449 } else if (mContinueSessionId != null) { 450 // create an in-memory derived plan that contains the notExecuted tests from previous 451 // session 452 // use timestamp as plan name so it will hopefully be unique 453 String uniquePlanName = Long.toString(System.currentTimeMillis()); 454 PlanCreator planCreator = new PlanCreator(uniquePlanName, mContinueSessionId, 455 CtsTestStatus.NOT_EXECUTED); 456 ITestPlan plan = createPlan(planCreator); 457 for (String uri : plan.getTestUris()) { 458 if (!mExcludedPackageNames.contains(uri)) { 459 ITestPackageDef testPackage = testRepo.getTestPackage(uri); 460 testPackage.setExcludedTestFilter(plan.getExcludedTestFilter(uri)); 461 testPkgDefs.add(testPackage); 462 } 463 } 464 } else { 465 // should never get here - was checkFields() not called? 466 throw new IllegalStateException("nothing to run?"); 467 } 468 return testPkgDefs; 469 } 470 471 /** 472 * Return the list of unique prerequisite Android package names 473 * @param testPackages 474 * @return 475 */ 476 private Collection<String> getPrerequisitePackageNames(List<TestPackage> testPackages) { 477 Set<String> pkgNames = new HashSet<String>(); 478 for (TestPackage testPkg : testPackages) { 479 String pkgName = testPkg.mPackageDef.getTargetPackageName(); 480 if (pkgName != null) { 481 pkgNames.add(pkgName); 482 } 483 } 484 return pkgNames; 485 } 486 487 /** 488 * Return the list of unique prerequisite apks to install 489 * @param testPackages 490 * @return 491 */ 492 private Collection<String> getPrerequisiteApks(List<TestPackage> testPackages) { 493 Set<String> apkNames = new HashSet<String>(); 494 for (TestPackage testPkg : testPackages) { 495 String apkName = testPkg.mPackageDef.getTargetApkName(); 496 if (apkName != null) { 497 apkNames.add(apkName); 498 } 499 } 500 return apkNames; 501 } 502 503 /** 504 * Install the collection of test apk file names 505 * 506 * @param prerequisiteApks 507 * @throws DeviceNotAvailableException 508 */ 509 private void installPrerequisiteApks(Collection<String> prerequisiteApks) 510 throws DeviceNotAvailableException { 511 for (String apkName : prerequisiteApks) { 512 try { 513 File apkFile = mCtsBuild.getTestApp(apkName); 514 String errorCode = getDevice().installPackage(apkFile, true); 515 if (errorCode != null) { 516 CLog.e("Failed to install %s. Reason: %s", apkName, errorCode); 517 } 518 } catch (FileNotFoundException e) { 519 CLog.e("Could not find test apk %s", apkName); 520 } 521 } 522 } 523 524 /** 525 * Uninstalls the collection of android package names from device. 526 * 527 * @param uninstallPackages 528 */ 529 private void uninstallPrequisiteApks(Collection<String> uninstallPackages) 530 throws DeviceNotAvailableException { 531 for (String pkgName : uninstallPackages) { 532 getDevice().uninstallPackage(pkgName); 533 } 534 } 535 536 /** 537 * {@inheritDoc} 538 */ 539 @Override 540 public Collection<IRemoteTest> split() { 541 if (mShards <= 1) { 542 return null; 543 } 544 checkFields(); 545 List<TestPackage> allTests = buildTestsToRun(); 546 547 if (allTests.size() <= 1) { 548 Log.w(LOG_TAG, "no tests to shard!"); 549 return null; 550 } 551 552 // treat shardQueue as a circular queue, to sequentially distribute tests among shards 553 Queue<IRemoteTest> shardQueue = new LinkedList<IRemoteTest>(); 554 // don't create more shards than the number of tests we have! 555 for (int i = 0; i < mShards && i < allTests.size(); i++) { 556 CtsTest shard = new CtsTest(); 557 shard.mRemainingTestPkgs = new LinkedList<TestPackage>(); 558 shardQueue.add(shard); 559 } 560 while (!allTests.isEmpty()) { 561 TestPackage testPair = allTests.remove(0); 562 CtsTest shard = (CtsTest)shardQueue.poll(); 563 shard.mRemainingTestPkgs.add(testPair); 564 shardQueue.add(shard); 565 } 566 return shardQueue; 567 } 568 569 /** 570 * Runs the device info collector instrumentation on device, and forwards it to test listeners 571 * as run metrics. 572 * <p/> 573 * Exposed so unit tests can mock. 574 * 575 * @param listeners 576 * @throws DeviceNotAvailableException 577 * @throws FileNotFoundException 578 */ 579 void collectDeviceInfo(ITestDevice device, CtsBuildHelper ctsBuild, 580 ITestInvocationListener listener) throws DeviceNotAvailableException { 581 if (!mSkipDeviceInfo) { 582 DeviceInfoCollector.collectDeviceInfo(device, ctsBuild.getTestCasesDir(), listener); 583 } 584 } 585 586 /** 587 * Factory method for creating a {@link ITestPackageRepo}. 588 * <p/> 589 * Exposed for unit testing 590 */ 591 ITestPackageRepo createTestCaseRepo() { 592 return new TestPackageRepo(mCtsBuild.getTestCasesDir()); 593 } 594 595 /** 596 * Factory method for creating a {@link TestPlan}. 597 * <p/> 598 * Exposed for unit testing 599 */ 600 ITestPlan createPlan(String planName) { 601 return new TestPlan(planName); 602 } 603 604 /** 605 * Factory method for creating a {@link TestPlan} from a {@link PlanCreator}. 606 * <p/> 607 * Exposed for unit testing 608 * @throws ConfigurationException 609 */ 610 ITestPlan createPlan(PlanCreator planCreator) throws ConfigurationException { 611 return planCreator.createDerivedPlan(mCtsBuild); 612 } 613 614 /** 615 * Factory method for creating a {@link InputStream} from a plan xml file. 616 * <p/> 617 * Exposed for unit testing 618 */ 619 InputStream createXmlStream(File xmlFile) throws FileNotFoundException { 620 return new BufferedInputStream(new FileInputStream(xmlFile)); 621 } 622 623 private void checkFields() { 624 // for simplicity of command line usage, make --plan, --package, and --class mutually 625 // exclusive 626 boolean mutualExclusiveArgs = xor(mPlanName != null, mPackageNames.size() > 0, 627 mClassName != null, mContinueSessionId != null); 628 629 if (!mutualExclusiveArgs) { 630 throw new IllegalArgumentException(String.format( 631 "Ambiguous or missing arguments. " + 632 "One and only one of --%s --%s(s), --%s or --%s to run can be specified", 633 PLAN_OPTION, PACKAGE_OPTION, CLASS_OPTION, CONTINUE_OPTION)); 634 } 635 if (mMethodName != null && mClassName == null) { 636 throw new IllegalArgumentException(String.format( 637 "Must specify --%s when --%s is used", CLASS_OPTION, METHOD_OPTION)); 638 } 639 if (mCtsBuild == null) { 640 throw new IllegalArgumentException("missing CTS build"); 641 } 642 if ("CTS".equals(mPlanName)) { 643 CLog.i("Switching to CTS-TF plan instead of CTS plan for faster execution of vm-tests"); 644 mPlanName = "CTS-TF"; 645 } 646 } 647 648 /** 649 * Helper method to perform exclusive or on list of boolean arguments 650 * 651 * @param args set of booleans on which to perform exclusive or 652 * @return <code>true</code> if one and only one of <var>args</code> is <code>true</code>. 653 * Otherwise return <code>false</code>. 654 */ 655 private boolean xor(boolean... args) { 656 boolean currentVal = args[0]; 657 for (int i=1; i < args.length; i++) { 658 if (currentVal && args[i]) { 659 return false; 660 } 661 currentVal |= args[i]; 662 } 663 return currentVal; 664 } 665 666 /** 667 * Forward the digest and package name to the listener as a metric 668 * 669 * @param listener 670 */ 671 private void forwardPackageDetails(ITestPackageDef def, ITestInvocationListener listener) { 672 Map<String, String> metrics = new HashMap<String, String>(2); 673 metrics.put(PACKAGE_NAME_METRIC, def.getName()); 674 metrics.put(PACKAGE_DIGEST_METRIC, def.getDigest()); 675 listener.testRunStarted(def.getUri(), 0); 676 listener.testRunEnded(0, metrics); 677 } 678 } 679