1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.cts.tradefed.testtype; 18 19 import com.android.cts.tradefed.build.CtsBuildHelper; 20 import com.android.cts.tradefed.device.DeviceInfoCollector; 21 import com.android.cts.tradefed.result.CtsTestStatus; 22 import com.android.cts.tradefed.result.PlanCreator; 23 import com.android.ddmlib.Log; 24 import com.android.ddmlib.Log.LogLevel; 25 import com.android.ddmlib.testrunner.TestIdentifier; 26 import com.android.tradefed.build.IBuildInfo; 27 import com.android.tradefed.config.ConfigurationException; 28 import com.android.tradefed.config.Option; 29 import com.android.tradefed.config.Option.Importance; 30 import com.android.tradefed.device.DeviceNotAvailableException; 31 import com.android.tradefed.device.ITestDevice; 32 import com.android.tradefed.device.TestDeviceOptions; 33 import com.android.tradefed.log.LogUtil.CLog; 34 import com.android.tradefed.result.ITestInvocationListener; 35 import com.android.tradefed.result.InputStreamSource; 36 import com.android.tradefed.result.LogDataType; 37 import com.android.tradefed.result.ResultForwarder; 38 import com.android.tradefed.testtype.IBuildReceiver; 39 import com.android.tradefed.testtype.IDeviceTest; 40 import com.android.tradefed.testtype.IRemoteTest; 41 import com.android.tradefed.testtype.IResumableTest; 42 import com.android.tradefed.testtype.IShardableTest; 43 import com.android.tradefed.util.RunUtil; 44 import com.android.tradefed.util.xml.AbstractXmlParser.ParseException; 45 46 import java.io.BufferedInputStream; 47 import java.io.File; 48 import java.io.FileInputStream; 49 import java.io.FileNotFoundException; 50 import java.io.InputStream; 51 import java.lang.InterruptedException; 52 import java.lang.System; 53 import java.lang.Thread; 54 import java.util.ArrayList; 55 import java.util.Arrays; 56 import java.util.Collection; 57 import java.util.HashMap; 58 import java.util.HashSet; 59 import java.util.LinkedHashSet; 60 import java.util.LinkedList; 61 import java.util.List; 62 import java.util.Map; 63 import java.util.Queue; 64 import java.util.Set; 65 66 import junit.framework.Test; 67 68 /** 69 * A {@link Test} for running CTS tests. 70 * <p/> 71 * Supports running all the tests contained in a CTS plan, or individual test packages. 72 */ 73 public class CtsTest implements IDeviceTest, IResumableTest, IShardableTest, IBuildReceiver { 74 private static final String LOG_TAG = "CtsTest"; 75 76 public static final String PLAN_OPTION = "plan"; 77 private static final String PACKAGE_OPTION = "package"; 78 private static final String CLASS_OPTION = "class"; 79 private static final String METHOD_OPTION = "method"; 80 public static final String CONTINUE_OPTION = "continue-session"; 81 public static final String RUN_KNOWN_FAILURES_OPTION = "run-known-failures"; 82 83 public static final String PACKAGE_NAME_METRIC = "packageName"; 84 public static final String PACKAGE_DIGEST_METRIC = "packageDigest"; 85 86 private ITestDevice mDevice; 87 88 @Option(name = PLAN_OPTION, description = "the test plan to run.", 89 importance = Importance.IF_UNSET) 90 private String mPlanName = null; 91 92 @Option(name = PACKAGE_OPTION, shortName = 'p', description = "the test packages(s) to run.", 93 importance = Importance.IF_UNSET) 94 private Collection<String> mPackageNames = new ArrayList<String>(); 95 96 @Option(name = "exclude-package", description = "the test packages(s) to exclude from the run.") 97 private Collection<String> mExcludedPackageNames = new ArrayList<String>(); 98 99 @Option(name = CLASS_OPTION, shortName = 'c', description = "run a specific test class.", 100 importance = Importance.IF_UNSET) 101 private String mClassName = null; 102 103 @Option(name = METHOD_OPTION, shortName = 'm', 104 description = "run a specific test method, from given --class.", 105 importance = Importance.IF_UNSET) 106 private String mMethodName = null; 107 108 @Option(name = CONTINUE_OPTION, 109 description = "continue a previous test session.", 110 importance = Importance.IF_UNSET) 111 private Integer mContinueSessionId = null; 112 113 @Option(name = "skip-device-info", shortName = 'd', description = 114 "flag to control whether to collect info from device. Providing this flag will speed up " + 115 "test execution for short test runs but will result in required data being omitted from " + 116 "the test report.") 117 private boolean mSkipDeviceInfo = false; 118 119 @Option(name = "resume", description = 120 "flag to attempt to automatically resume aborted test run on another connected device. ") 121 private boolean mResume = false; 122 123 @Option(name = "shards", description = 124 "shard the tests to run into separately runnable chunks to execute on multiple devices " + 125 "concurrently.") 126 private int mShards = 1; 127 128 @Option(name = "screenshot", description = 129 "flag for taking a screenshot of the device when test execution is complete.") 130 private boolean mScreenshot = false; 131 132 @Option(name = "bugreport", shortName = 'b', description = 133 "take a bugreport after each failed test. " + 134 "Warning: can potentially use a lot of disk space.") 135 private boolean mBugreport = false; 136 137 @Option(name = RUN_KNOWN_FAILURES_OPTION, shortName = 'k', description = 138 "run tests including known failures") 139 private boolean mIncludeKnownFailures; 140 141 @Option(name = "disable-reboot", description = 142 "Do not reboot device after running some amount of tests. Default behavior is to reboot.") 143 private boolean mDisableReboot = false; 144 145 @Option(name = "reboot-wait-time", description = 146 "Additional wait time in ms after boot complete.") 147 private int mRebootWaitTimeMSec = 2 * 60 * 1000; 148 149 @Option(name = "reboot-interval", description = 150 "Interval between each reboot in min.") 151 private int mRebootIntervalMin = 30; 152 153 @Option(name = "screenshot-on-failure", description = 154 "take a screenshot on every test failure.") 155 private boolean mScreenshotOnFailures = false; 156 157 @Option(name = "logcat-on-failure", description = 158 "take a logcat snapshot on every test failure. Unlike --bugreport, this can capture" + 159 "logs even if connection with device has been lost, as well as being much more " + 160 "performant.") 161 private boolean mLogcatOnFailures = false; 162 163 @Option(name = "logcat-on-failure-size", description = 164 "The max number of logcat data in bytes to capture when --logcat-on-failure is on. " + 165 "Should be an amount that can comfortably fit in memory.") 166 private int mMaxLogcatBytes = 500 * 1024; // 500K 167 168 private long mPrevRebootTime; // last reboot time 169 170 /** data structure for a {@link IRemoteTest} and its known tests */ 171 class TestPackage { 172 private final IRemoteTest mTestForPackage; 173 private final Collection<TestIdentifier> mKnownTests; 174 private final ITestPackageDef mPackageDef; 175 176 TestPackage(ITestPackageDef packageDef, IRemoteTest testForPackage, 177 Collection<TestIdentifier> knownTests) { 178 mPackageDef = packageDef; 179 mTestForPackage = testForPackage; 180 mKnownTests = knownTests; 181 } 182 183 IRemoteTest getTestForPackage() { 184 return mTestForPackage; 185 } 186 187 Collection<TestIdentifier> getKnownTests() { 188 return mKnownTests; 189 } 190 191 ITestPackageDef getPackageDef() { 192 return mPackageDef; 193 } 194 195 /** 196 * Return the test run name that should be used for the TestPackage 197 */ 198 String getTestRunName() { 199 return mPackageDef.getUri(); 200 } 201 } 202 203 /** 204 * A {@link ResultForwarder} that will forward a bugreport on each failed test. 205 */ 206 private static class FailedTestBugreportGenerator extends ResultForwarder { 207 private ITestDevice mDevice; 208 209 public FailedTestBugreportGenerator(ITestInvocationListener listener, ITestDevice device) { 210 super(listener); 211 mDevice = device; 212 } 213 214 @Override 215 public void testFailed(TestFailure status, TestIdentifier test, String trace) { 216 super.testFailed(status, test, trace); 217 InputStreamSource bugSource = mDevice.getBugreport(); 218 super.testLog(String.format("bug-%s_%s", test.getClassName(), test.getTestName()), 219 LogDataType.TEXT, bugSource); 220 bugSource.cancel(); 221 } 222 } 223 224 /** 225 * A {@link ResultForwarder} that will forward a logcat snapshot on each failed test. 226 */ 227 private static class FailedTestLogcatGenerator extends ResultForwarder { 228 private ITestDevice mDevice; 229 private int mNumLogcatBytes; 230 231 public FailedTestLogcatGenerator(ITestInvocationListener listener, ITestDevice device, 232 int maxLogcatBytes) { 233 super(listener); 234 mDevice = device; 235 mNumLogcatBytes = maxLogcatBytes; 236 } 237 238 @Override 239 public void testFailed(TestFailure status, TestIdentifier test, String trace) { 240 super.testFailed(status, test, trace); 241 // sleep a small amount of time to ensure test failure stack trace makes it into logcat 242 // capture 243 RunUtil.getDefault().sleep(10); 244 InputStreamSource logSource = mDevice.getLogcat(mNumLogcatBytes); 245 super.testLog(String.format("logcat-%s_%s", test.getClassName(), test.getTestName()), 246 LogDataType.TEXT, logSource); 247 logSource.cancel(); 248 } 249 } 250 251 /** 252 * A {@link ResultForwarder} that will forward a screenshot on test failures. 253 */ 254 private static class FailedTestScreenshotGenerator extends ResultForwarder { 255 private ITestDevice mDevice; 256 257 public FailedTestScreenshotGenerator(ITestInvocationListener listener, 258 ITestDevice device) { 259 super(listener); 260 mDevice = device; 261 } 262 263 @Override 264 public void testFailed(TestFailure status, TestIdentifier test, String trace) { 265 super.testFailed(status, test, trace); 266 267 try { 268 InputStreamSource screenSource = mDevice.getScreenshot(); 269 super.testLog(String.format("screenshot-%s_%s", test.getClassName(), 270 test.getTestName()), LogDataType.PNG, screenSource); 271 screenSource.cancel(); 272 } catch (DeviceNotAvailableException e) { 273 // TODO: rethrow this somehow 274 CLog.e("Device %s became unavailable while capturing screenshot, %s", 275 mDevice.getSerialNumber(), e.toString()); 276 } 277 } 278 } 279 280 /** list of remaining tests to execute */ 281 private List<TestPackage> mRemainingTestPkgs = null; 282 283 private CtsBuildHelper mCtsBuild = null; 284 private IBuildInfo mBuildInfo = null; 285 286 /** 287 * {@inheritDoc} 288 */ 289 @Override 290 public ITestDevice getDevice() { 291 return mDevice; 292 } 293 294 /** 295 * {@inheritDoc} 296 */ 297 @Override 298 public void setDevice(ITestDevice device) { 299 mDevice = device; 300 } 301 302 /** 303 * Set the plan name to run. 304 * <p/> 305 * Exposed for unit testing 306 */ 307 void setPlanName(String planName) { 308 mPlanName = planName; 309 } 310 311 /** 312 * Set the skip collect device info flag. 313 * <p/> 314 * Exposed for unit testing 315 */ 316 void setSkipDeviceInfo(boolean skipDeviceInfo) { 317 mSkipDeviceInfo = skipDeviceInfo; 318 } 319 320 /** 321 * Adds a package name to the list of test packages to run. 322 * <p/> 323 * Exposed for unit testing 324 */ 325 void addPackageName(String packageName) { 326 mPackageNames.add(packageName); 327 } 328 329 /** 330 * Adds a package name to the list of test packages to exclude. 331 * <p/> 332 * Exposed for unit testing 333 */ 334 void addExcludedPackageName(String packageName) { 335 mExcludedPackageNames.add(packageName); 336 } 337 338 /** 339 * Set the test class name to run. 340 * <p/> 341 * Exposed for unit testing 342 */ 343 void setClassName(String className) { 344 mClassName = className; 345 } 346 347 /** 348 * Set the test method name to run. 349 * <p/> 350 * Exposed for unit testing 351 */ 352 void setMethodName(String methodName) { 353 mMethodName = methodName; 354 } 355 356 /** 357 * Sets the test session id to continue. 358 * <p/> 359 * Exposed for unit testing 360 */ 361 void setContinueSessionId(int sessionId) { 362 mContinueSessionId = sessionId; 363 } 364 365 /** 366 * {@inheritDoc} 367 */ 368 @Override 369 public boolean isResumable() { 370 return mResume; 371 } 372 373 /** 374 * {@inheritDoc} 375 */ 376 @Override 377 public void setBuild(IBuildInfo build) { 378 mCtsBuild = CtsBuildHelper.createBuildHelper(build); 379 mBuildInfo = build; 380 } 381 382 /** 383 * Set the CTS build container. 384 * <p/> 385 * Exposed so unit tests can mock the provided build. 386 * 387 * @param buildHelper 388 */ 389 void setBuildHelper(CtsBuildHelper buildHelper) { 390 mCtsBuild = buildHelper; 391 } 392 393 /** 394 * {@inheritDoc} 395 */ 396 @Override 397 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 398 if (getDevice() == null) { 399 throw new IllegalArgumentException("missing device"); 400 } 401 402 if (mRemainingTestPkgs == null) { 403 checkFields(); 404 mRemainingTestPkgs = buildTestsToRun(); 405 } 406 if (mBugreport) { 407 FailedTestBugreportGenerator bugListener = new FailedTestBugreportGenerator(listener, 408 getDevice()); 409 listener = bugListener; 410 } 411 if (mScreenshotOnFailures) { 412 FailedTestScreenshotGenerator screenListener = new FailedTestScreenshotGenerator( 413 listener, getDevice()); 414 listener = screenListener; 415 } 416 if (mLogcatOnFailures) { 417 FailedTestLogcatGenerator logcatListener = new FailedTestLogcatGenerator( 418 listener, getDevice(), mMaxLogcatBytes); 419 listener = logcatListener; 420 } 421 422 // collect and install the prerequisiteApks first, to save time when multiple test 423 // packages are using the same prerequisite apk (I'm looking at you, CtsTestStubs!) 424 Collection<String> prerequisiteApks = getPrerequisiteApks(mRemainingTestPkgs); 425 Collection<String> uninstallPackages = getPrerequisitePackageNames(mRemainingTestPkgs); 426 ResultFilter filter = new ResultFilter(listener, mRemainingTestPkgs); 427 428 try { 429 installPrerequisiteApks(prerequisiteApks); 430 431 // always collect the device info, even for resumed runs, since test will likely be 432 // running on a different device 433 collectDeviceInfo(getDevice(), mCtsBuild, listener); 434 if (mRemainingTestPkgs.size() > 1 && !mDisableReboot) { 435 Log.i(LOG_TAG, "Initial reboot for multiple packages"); 436 rebootDevice(); 437 } 438 mPrevRebootTime = System.currentTimeMillis(); 439 440 while (!mRemainingTestPkgs.isEmpty()) { 441 TestPackage knownTests = mRemainingTestPkgs.get(0); 442 443 IRemoteTest test = knownTests.getTestForPackage(); 444 if (test instanceof IDeviceTest) { 445 ((IDeviceTest)test).setDevice(getDevice()); 446 } 447 if (test instanceof IBuildReceiver) { 448 ((IBuildReceiver)test).setBuild(mBuildInfo); 449 } 450 451 forwardPackageDetails(knownTests.getPackageDef(), listener); 452 test.run(filter); 453 mRemainingTestPkgs.remove(0); 454 if (mRemainingTestPkgs.size() > 0) { 455 rebootIfNecessary(knownTests, mRemainingTestPkgs.get(0)); 456 // remove artifacts like status bar from the previous test. 457 // But this cannot dismiss dialog popped-up. 458 changeToHomeScreen(); 459 } 460 } 461 462 if (mScreenshot) { 463 InputStreamSource screenshotSource = getDevice().getScreenshot(); 464 try { 465 listener.testLog("screenshot", LogDataType.PNG, screenshotSource); 466 } finally { 467 screenshotSource.cancel(); 468 } 469 } 470 471 uninstallPrequisiteApks(uninstallPackages); 472 473 } finally { 474 filter.reportUnexecutedTests(); 475 } 476 } 477 478 private void rebootIfNecessary(TestPackage testFinished, TestPackage testToRun) 479 throws DeviceNotAvailableException { 480 // If there comes spurious failure like INJECT_EVENTS for a package, 481 // reboot it before running it. 482 // Also reboot after package which is know to leave pop-up behind 483 final List<String> rebootAfterList = Arrays.asList( 484 "CtsMediaTestCases", 485 "CtsAccessibilityTestCases"); 486 final List<String> rebootBeforeList = Arrays.asList( 487 "CtsAnimationTestCases", 488 "CtsGraphicsTestCases", 489 "CtsViewTestCases", 490 "CtsWidgetTestCases" ); 491 long intervalInMSec = mRebootIntervalMin * 60 * 1000; 492 if (mDevice.getSerialNumber().startsWith("emulator-")) { 493 return; 494 } 495 if (!mDisableReboot) { 496 long currentTime = System.currentTimeMillis(); 497 if (((currentTime - mPrevRebootTime) > intervalInMSec) || 498 rebootAfterList.contains(testFinished.getPackageDef().getName()) || 499 rebootBeforeList.contains(testToRun.getPackageDef().getName()) ) { 500 Log.i(LOG_TAG, 501 String.format("Rebooting after running package %s, before package %s", 502 testFinished.getPackageDef().getName(), 503 testToRun.getPackageDef().getName())); 504 rebootDevice(); 505 mPrevRebootTime = System.currentTimeMillis(); 506 } 507 } 508 } 509 510 private void rebootDevice() throws DeviceNotAvailableException { 511 final int TIMEOUT_MS = 10 * 60 * 1000; 512 TestDeviceOptions options = mDevice.getOptions(); 513 // store default value and increase time-out for reboot 514 int rebootTimeout = options.getRebootTimeout(); 515 long onlineTimeout = options.getOnlineTimeout(); 516 options.setRebootTimeout(TIMEOUT_MS); 517 options.setOnlineTimeout(TIMEOUT_MS); 518 mDevice.setOptions(options); 519 520 mDevice.reboot(); 521 522 // restore default values 523 options.setRebootTimeout(rebootTimeout); 524 options.setOnlineTimeout(onlineTimeout); 525 mDevice.setOptions(options); 526 Log.i(LOG_TAG, "Rebooting done"); 527 try { 528 Thread.sleep(mRebootWaitTimeMSec); 529 } catch (InterruptedException e) { 530 Log.i(LOG_TAG, "Boot wait interrupted"); 531 } 532 } 533 534 private void changeToHomeScreen() throws DeviceNotAvailableException { 535 final String homeCmd = "input keyevent 3"; 536 537 mDevice.executeShellCommand(homeCmd); 538 try { 539 Thread.sleep(1000); 540 } catch (InterruptedException e) { 541 //ignore 542 } 543 } 544 /** 545 * Build the list of test packages to run 546 */ 547 private List<TestPackage> buildTestsToRun() { 548 List<TestPackage> testPkgList = new LinkedList<TestPackage>(); 549 try { 550 ITestPackageRepo testRepo = createTestCaseRepo(); 551 Collection<ITestPackageDef> testPkgDefs = getTestPackagesToRun(testRepo); 552 553 for (ITestPackageDef testPkgDef : testPkgDefs) { 554 addTestPackage(testPkgList, testPkgDef); 555 } 556 if (testPkgList.isEmpty()) { 557 Log.logAndDisplay(LogLevel.WARN, LOG_TAG, "No tests to run"); 558 } 559 } catch (FileNotFoundException e) { 560 throw new IllegalArgumentException("failed to find CTS plan file", e); 561 } catch (ParseException e) { 562 throw new IllegalArgumentException("failed to parse CTS plan file", e); 563 } catch (ConfigurationException e) { 564 throw new IllegalArgumentException("failed to process arguments", e); 565 } 566 return testPkgList; 567 } 568 569 /** 570 * Adds a test package to the list of packages to test 571 * 572 * @param testList 573 * @param testPkgDef 574 */ 575 private void addTestPackage(List<TestPackage> testList, ITestPackageDef testPkgDef) { 576 IRemoteTest testForPackage = testPkgDef.createTest(mCtsBuild.getTestCasesDir()); 577 if (testForPackage != null) { 578 Collection<TestIdentifier> knownTests = testPkgDef.getTests(); 579 testList.add(new TestPackage(testPkgDef, testForPackage, knownTests)); 580 } 581 } 582 583 /** 584 * Return the list of test package defs to run 585 * 586 * @return the list of test package defs to run 587 * @throws ParseException 588 * @throws FileNotFoundException 589 * @throws ConfigurationException 590 */ 591 private Collection<ITestPackageDef> getTestPackagesToRun(ITestPackageRepo testRepo) 592 throws ParseException, FileNotFoundException, ConfigurationException { 593 // use LinkedHashSet to have predictable iteration order 594 Set<ITestPackageDef> testPkgDefs = new LinkedHashSet<ITestPackageDef>(); 595 if (mPlanName != null) { 596 Log.i(LOG_TAG, String.format("Executing CTS test plan %s", mPlanName)); 597 File ctsPlanFile = mCtsBuild.getTestPlanFile(mPlanName); 598 ITestPlan plan = createPlan(mPlanName); 599 plan.parse(createXmlStream(ctsPlanFile)); 600 for (String uri : plan.getTestUris()) { 601 if (!mExcludedPackageNames.contains(uri)) { 602 ITestPackageDef testPackage = testRepo.getTestPackage(uri); 603 testPackage.setExcludedTestFilter(plan.getExcludedTestFilter(uri)); 604 testPkgDefs.add(testPackage); 605 } 606 } 607 } else if (mPackageNames.size() > 0){ 608 Log.i(LOG_TAG, String.format("Executing CTS test packages %s", mPackageNames)); 609 for (String uri : mPackageNames) { 610 ITestPackageDef testPackage = testRepo.getTestPackage(uri); 611 if (testPackage != null) { 612 testPkgDefs.add(testPackage); 613 } else { 614 throw new IllegalArgumentException(String.format( 615 "Could not find test package %s. " + 616 "Use 'list packages' to see available packages." , uri)); 617 } 618 } 619 } else if (mClassName != null) { 620 Log.i(LOG_TAG, String.format("Executing CTS test class %s", mClassName)); 621 // try to find package to run from class name 622 String packageUri = testRepo.findPackageForTest(mClassName); 623 if (packageUri != null) { 624 ITestPackageDef testPackageDef = testRepo.getTestPackage(packageUri); 625 testPackageDef.setClassName(mClassName, mMethodName); 626 testPkgDefs.add(testPackageDef); 627 } else { 628 Log.logAndDisplay(LogLevel.WARN, LOG_TAG, String.format( 629 "Could not find package for test class %s", mClassName)); 630 } 631 } else if (mContinueSessionId != null) { 632 // create an in-memory derived plan that contains the notExecuted tests from previous 633 // session 634 // use timestamp as plan name so it will hopefully be unique 635 String uniquePlanName = Long.toString(System.currentTimeMillis()); 636 PlanCreator planCreator = new PlanCreator(uniquePlanName, mContinueSessionId, 637 CtsTestStatus.NOT_EXECUTED); 638 ITestPlan plan = createPlan(planCreator); 639 for (String uri : plan.getTestUris()) { 640 if (!mExcludedPackageNames.contains(uri)) { 641 ITestPackageDef testPackage = testRepo.getTestPackage(uri); 642 testPackage.setExcludedTestFilter(plan.getExcludedTestFilter(uri)); 643 testPkgDefs.add(testPackage); 644 } 645 } 646 } else { 647 // should never get here - was checkFields() not called? 648 throw new IllegalStateException("nothing to run?"); 649 } 650 return testPkgDefs; 651 } 652 653 /** 654 * Return the list of unique prerequisite Android package names 655 * @param testPackages 656 */ 657 private Collection<String> getPrerequisitePackageNames(List<TestPackage> testPackages) { 658 Set<String> pkgNames = new HashSet<String>(); 659 for (TestPackage testPkg : testPackages) { 660 String pkgName = testPkg.mPackageDef.getTargetPackageName(); 661 if (pkgName != null) { 662 pkgNames.add(pkgName); 663 } 664 } 665 return pkgNames; 666 } 667 668 /** 669 * Return the list of unique prerequisite apks to install 670 * @param testPackages 671 */ 672 private Collection<String> getPrerequisiteApks(List<TestPackage> testPackages) { 673 Set<String> apkNames = new HashSet<String>(); 674 for (TestPackage testPkg : testPackages) { 675 String apkName = testPkg.mPackageDef.getTargetApkName(); 676 if (apkName != null) { 677 apkNames.add(apkName); 678 } 679 } 680 return apkNames; 681 } 682 683 /** 684 * Install the collection of test apk file names 685 * 686 * @param prerequisiteApks 687 * @throws DeviceNotAvailableException 688 */ 689 private void installPrerequisiteApks(Collection<String> prerequisiteApks) 690 throws DeviceNotAvailableException { 691 for (String apkName : prerequisiteApks) { 692 try { 693 File apkFile = mCtsBuild.getTestApp(apkName); 694 String errorCode = getDevice().installPackage(apkFile, true); 695 if (errorCode != null) { 696 CLog.e("Failed to install %s. Reason: %s", apkName, errorCode); 697 } 698 } catch (FileNotFoundException e) { 699 CLog.e("Could not find test apk %s", apkName); 700 } 701 } 702 } 703 704 /** 705 * Uninstalls the collection of android package names from device. 706 * 707 * @param uninstallPackages 708 */ 709 private void uninstallPrequisiteApks(Collection<String> uninstallPackages) 710 throws DeviceNotAvailableException { 711 for (String pkgName : uninstallPackages) { 712 getDevice().uninstallPackage(pkgName); 713 } 714 } 715 716 /** 717 * {@inheritDoc} 718 */ 719 @Override 720 public Collection<IRemoteTest> split() { 721 if (mShards <= 1) { 722 return null; 723 } 724 checkFields(); 725 List<TestPackage> allTests = buildTestsToRun(); 726 727 if (allTests.size() <= 1) { 728 Log.w(LOG_TAG, "no tests to shard!"); 729 return null; 730 } 731 732 // treat shardQueue as a circular queue, to sequentially distribute tests among shards 733 Queue<IRemoteTest> shardQueue = new LinkedList<IRemoteTest>(); 734 // don't create more shards than the number of tests we have! 735 for (int i = 0; i < mShards && i < allTests.size(); i++) { 736 CtsTest shard = new CtsTest(); 737 shard.mRemainingTestPkgs = new LinkedList<TestPackage>(); 738 shardQueue.add(shard); 739 } 740 while (!allTests.isEmpty()) { 741 TestPackage testPair = allTests.remove(0); 742 CtsTest shard = (CtsTest)shardQueue.poll(); 743 shard.mRemainingTestPkgs.add(testPair); 744 shardQueue.add(shard); 745 } 746 return shardQueue; 747 } 748 749 /** 750 * Runs the device info collector instrumentation on device, and forwards it to test listeners 751 * as run metrics. 752 * <p/> 753 * Exposed so unit tests can mock. 754 * 755 * @throws DeviceNotAvailableException 756 */ 757 void collectDeviceInfo(ITestDevice device, CtsBuildHelper ctsBuild, 758 ITestInvocationListener listener) throws DeviceNotAvailableException { 759 if (!mSkipDeviceInfo) { 760 DeviceInfoCollector.collectDeviceInfo(device, ctsBuild.getTestCasesDir(), listener); 761 } 762 } 763 764 /** 765 * Factory method for creating a {@link ITestPackageRepo}. 766 * <p/> 767 * Exposed for unit testing 768 */ 769 ITestPackageRepo createTestCaseRepo() { 770 return new TestPackageRepo(mCtsBuild.getTestCasesDir(), mIncludeKnownFailures); 771 } 772 773 /** 774 * Factory method for creating a {@link TestPlan}. 775 * <p/> 776 * Exposed for unit testing 777 */ 778 ITestPlan createPlan(String planName) { 779 return new TestPlan(planName); 780 } 781 782 /** 783 * Factory method for creating a {@link TestPlan} from a {@link PlanCreator}. 784 * <p/> 785 * Exposed for unit testing 786 * @throws ConfigurationException 787 */ 788 ITestPlan createPlan(PlanCreator planCreator) throws ConfigurationException { 789 return planCreator.createDerivedPlan(mCtsBuild); 790 } 791 792 /** 793 * Factory method for creating a {@link InputStream} from a plan xml file. 794 * <p/> 795 * Exposed for unit testing 796 */ 797 InputStream createXmlStream(File xmlFile) throws FileNotFoundException { 798 return new BufferedInputStream(new FileInputStream(xmlFile)); 799 } 800 801 private void checkFields() { 802 // for simplicity of command line usage, make --plan, --package, and --class mutually 803 // exclusive 804 boolean mutualExclusiveArgs = xor(mPlanName != null, mPackageNames.size() > 0, 805 mClassName != null, mContinueSessionId != null); 806 807 if (!mutualExclusiveArgs) { 808 throw new IllegalArgumentException(String.format( 809 "Ambiguous or missing arguments. " + 810 "One and only one of --%s --%s(s), --%s or --%s to run can be specified", 811 PLAN_OPTION, PACKAGE_OPTION, CLASS_OPTION, CONTINUE_OPTION)); 812 } 813 if (mMethodName != null && mClassName == null) { 814 throw new IllegalArgumentException(String.format( 815 "Must specify --%s when --%s is used", CLASS_OPTION, METHOD_OPTION)); 816 } 817 if (mCtsBuild == null) { 818 throw new IllegalArgumentException("missing CTS build"); 819 } 820 } 821 822 /** 823 * Helper method to perform exclusive or on list of boolean arguments 824 * 825 * @param args set of booleans on which to perform exclusive or 826 * @return <code>true</code> if one and only one of <var>args</code> is <code>true</code>. 827 * Otherwise return <code>false</code>. 828 */ 829 private boolean xor(boolean... args) { 830 boolean currentVal = args[0]; 831 for (int i=1; i < args.length; i++) { 832 if (currentVal && args[i]) { 833 return false; 834 } 835 currentVal |= args[i]; 836 } 837 return currentVal; 838 } 839 840 /** 841 * Forward the digest and package name to the listener as a metric 842 * 843 * @param listener 844 */ 845 private void forwardPackageDetails(ITestPackageDef def, ITestInvocationListener listener) { 846 Map<String, String> metrics = new HashMap<String, String>(2); 847 metrics.put(PACKAGE_NAME_METRIC, def.getName()); 848 metrics.put(PACKAGE_DIGEST_METRIC, def.getDigest()); 849 listener.testRunStarted(def.getUri(), 0); 850 listener.testRunEnded(0, metrics); 851 } 852 } 853
