1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.cts.tradefed.testtype; 18 19 import com.android.cts.tradefed.build.CtsBuildHelper; 20 import com.android.cts.tradefed.device.DeviceInfoCollector; 21 import com.android.cts.tradefed.result.CtsTestStatus; 22 import com.android.cts.tradefed.result.PlanCreator; 23 import com.android.cts.util.AbiUtils; 24 import com.android.ddmlib.Log; 25 import com.android.ddmlib.Log.LogLevel; 26 import com.android.ddmlib.testrunner.TestIdentifier; 27 import com.android.tradefed.build.IBuildInfo; 28 import com.android.tradefed.config.ConfigurationException; 29 import com.android.tradefed.config.Option; 30 import com.android.tradefed.config.Option.Importance; 31 import com.android.tradefed.config.OptionCopier; 32 import com.android.tradefed.device.DeviceNotAvailableException; 33 import com.android.tradefed.device.ITestDevice; 34 import com.android.tradefed.device.TestDeviceOptions; 35 import com.android.tradefed.log.LogUtil.CLog; 36 import com.android.tradefed.result.ITestInvocationListener; 37 import com.android.tradefed.result.InputStreamSource; 38 import com.android.tradefed.result.LogDataType; 39 import com.android.tradefed.result.ResultForwarder; 40 import com.android.tradefed.testtype.IAbi; 41 import com.android.tradefed.testtype.IBuildReceiver; 42 import com.android.tradefed.testtype.IDeviceTest; 43 import com.android.tradefed.testtype.IRemoteTest; 44 import com.android.tradefed.testtype.IResumableTest; 45 import com.android.tradefed.testtype.IShardableTest; 46 import com.android.tradefed.util.AbiFormatter; 47 import com.android.tradefed.util.RunUtil; 48 import com.android.tradefed.util.xml.AbstractXmlParser.ParseException; 49 50 import junit.framework.Test; 51 52 import java.io.BufferedInputStream; 53 import java.io.File; 54 import java.io.FileInputStream; 55 import java.io.FileNotFoundException; 56 import java.io.InputStream; 57 import java.util.ArrayList; 58 import java.util.Arrays; 59 import java.util.Collection; 60 import java.util.Collections; 61 import java.util.HashMap; 62 import java.util.HashSet; 63 import java.util.LinkedHashSet; 64 import java.util.LinkedList; 65 import java.util.List; 66 import java.util.Map; 67 import java.util.Set; 68 69 70 /** 71 * A {@link Test} for running CTS tests. 72 * <p/> 73 * Supports running all the tests contained in a CTS plan, or individual test packages. 74 */ 75 public class CtsTest implements IDeviceTest, IResumableTest, IShardableTest, IBuildReceiver { 76 private static final String LOG_TAG = "CtsTest"; 77 78 public static final String PLAN_OPTION = "plan"; 79 private static final String PACKAGE_OPTION = "package"; 80 private static final String CLASS_OPTION = "class"; 81 private static final String METHOD_OPTION = "method"; 82 private static final String TEST_OPTION = "test"; 83 public static final String CONTINUE_OPTION = "continue-session"; 84 public static final String RUN_KNOWN_FAILURES_OPTION = "run-known-failures"; 85 86 public static final String PACKAGE_NAME_METRIC = "packageName"; 87 public static final String PACKAGE_ABI_METRIC = "packageAbi"; 88 public static final String PACKAGE_DIGEST_METRIC = "packageDigest"; 89 90 @Option(name = PLAN_OPTION, description = "the test plan to run.", 91 importance = Importance.IF_UNSET) 92 private String mPlanName = null; 93 94 @Option(name = PACKAGE_OPTION, shortName = 'p', description = "the test packages(s) to run.", 95 importance = Importance.IF_UNSET) 96 private Collection<String> mPackageNames = new ArrayList<String>(); 97 98 @Option(name = "exclude-package", description = "the test packages(s) to exclude from the run.") 99 private Collection<String> mExcludedPackageNames = new ArrayList<String>(); 100 101 @Option(name = CLASS_OPTION, shortName = 'c', description = "run a specific test class.", 102 importance = Importance.IF_UNSET) 103 private String mClassName = null; 104 105 @Option(name = METHOD_OPTION, shortName = 'm', 106 description = "run a specific test method, from given --class.", 107 importance = Importance.IF_UNSET) 108 private String mMethodName = null; 109 110 @Option(name = TEST_OPTION, shortName = 't', description = "run a specific test", 111 importance = Importance.IF_UNSET) 112 private String mTestName = null; 113 114 @Option(name = CONTINUE_OPTION, 115 description = "continue a previous test session.", 116 importance = Importance.IF_UNSET) 117 private Integer mContinueSessionId = null; 118 119 @Option(name = "skip-device-info", shortName = 'd', description = 120 "flag to control whether to collect info from device. Providing this flag will speed up " + 121 "test execution for short test runs but will result in required data being omitted from " + 122 "the test report.") 123 private boolean mSkipDeviceInfo = false; 124 125 @Option(name = "resume", description = 126 "flag to attempt to automatically resume aborted test run on another connected device. ") 127 private boolean mResume = false; 128 129 @Option(name = "shards", description = 130 "shard the tests to run into separately runnable chunks to execute on multiple devices " + 131 "concurrently.") 132 private int mShards = 1; 133 134 @Option(name = "screenshot", description = 135 "flag for taking a screenshot of the device when test execution is complete.") 136 private boolean mScreenshot = false; 137 138 @Option(name = "bugreport", shortName = 'b', description = 139 "take a bugreport after each failed test. " + 140 "Warning: can potentially use a lot of disk space.") 141 private boolean mBugreport = false; 142 143 @Option(name = RUN_KNOWN_FAILURES_OPTION, shortName = 'k', description = 144 "run tests including known failures") 145 private boolean mIncludeKnownFailures; 146 147 @Option(name = "disable-reboot", description = 148 "Do not reboot device after running some amount of tests. Default behavior is to reboot.") 149 private boolean mDisableReboot = false; 150 151 @Option(name = "reboot-wait-time", description = 152 "Additional wait time in ms after boot complete.") 153 private int mRebootWaitTimeMSec = 2 * 60 * 1000; 154 155 @Option(name = "reboot-interval", description = 156 "Interval between each reboot in min.") 157 private int mRebootIntervalMin = 30; 158 159 @Option(name = "screenshot-on-failure", description = 160 "take a screenshot on every test failure.") 161 private boolean mScreenshotOnFailures = false; 162 163 @Option(name = "logcat-on-failure", description = 164 "take a logcat snapshot on every test failure. Unlike --bugreport, this can capture" + 165 "logs even if connection with device has been lost, as well as being much more " + 166 "performant.") 167 private boolean mLogcatOnFailures = false; 168 169 @Option(name = AbiFormatter.FORCE_ABI_STRING, 170 description = AbiFormatter.FORCE_ABI_DESCRIPTION, 171 importance = Importance.IF_UNSET) 172 private String mForceAbi = null; 173 174 @Option(name = "logcat-on-failure-size", description = 175 "The max number of logcat data in bytes to capture when --logcat-on-failure is on. " + 176 "Should be an amount that can comfortably fit in memory.") 177 private int mMaxLogcatBytes = 500 * 1024; // 500K 178 179 @Option(name = "collect-deqp-logs", description = 180 "Collect dEQP logs from the device.") 181 private boolean mCollectDeqpLogs = false; 182 183 @Option(name = "min-pre-reboot-package-count", description = 184 "The minimum number of packages to require a pre test reboot") 185 186 private int mMinPreRebootPackageCount = 2; 187 private final int mShardAssignment; 188 private final int mTotalShards; 189 private ITestDevice mDevice = null; 190 private CtsBuildHelper mCtsBuild = null; 191 private IBuildInfo mBuildInfo = null; 192 // last reboot time 193 private long mPrevRebootTime; 194 // The list of packages to run. populated in {@code setupTestPackageList} 195 // This is a member variable so that run can be called more than once 196 // and the test run is resumed. 197 private List<TestPackage> mTestPackageList = new ArrayList<>(); 198 // The index in the pacakge list of the last test to complete 199 private int mLastTestPackageIndex = 0; 200 201 /** data structure for a {@link IRemoteTest} and its known tests */ 202 static class TestPackage { 203 private final IRemoteTest mTestForPackage; 204 private final ITestPackageDef mPackageDef; 205 private final Collection<TestIdentifier> mKnownTests; 206 207 TestPackage(ITestPackageDef packageDef, IRemoteTest testForPackage) { 208 mPackageDef = packageDef; 209 mTestForPackage = testForPackage; 210 mKnownTests = packageDef.getTests(); 211 } 212 213 IRemoteTest getTestForPackage() { 214 return mTestForPackage; 215 } 216 217 Collection<TestIdentifier> getKnownTests() { 218 return mKnownTests; 219 } 220 221 ITestPackageDef getPackageDef() { 222 return mPackageDef; 223 } 224 225 /** 226 * @return the test run name that should be used for the TestPackage. 227 */ 228 String getTestRunName() { 229 return mPackageDef.getId(); 230 } 231 232 /** 233 * @return the ABI on which the test will run. 234 */ 235 IAbi getAbi() { 236 return mPackageDef.getAbi(); 237 } 238 } 239 240 /** 241 * A {@link ResultForwarder} that will forward a bugreport on each failed test. 242 */ 243 private static class FailedTestBugreportGenerator extends ResultForwarder { 244 private ITestDevice mDevice; 245 246 public FailedTestBugreportGenerator(ITestInvocationListener listener, ITestDevice device) { 247 super(listener); 248 mDevice = device; 249 } 250 251 @Override 252 public void testFailed(TestIdentifier test, String trace) { 253 super.testFailed(test, trace); 254 InputStreamSource bugSource = mDevice.getBugreport(); 255 super.testLog(String.format("bug-%s_%s", test.getClassName(), test.getTestName()), 256 LogDataType.TEXT, bugSource); 257 bugSource.cancel(); 258 } 259 } 260 261 /** 262 * A {@link ResultForwarder} that will forward a logcat snapshot on each failed test. 263 */ 264 private static class FailedTestLogcatGenerator extends ResultForwarder { 265 private ITestDevice mDevice; 266 private int mNumLogcatBytes; 267 268 public FailedTestLogcatGenerator(ITestInvocationListener listener, ITestDevice device, 269 int maxLogcatBytes) { 270 super(listener); 271 mDevice = device; 272 mNumLogcatBytes = maxLogcatBytes; 273 } 274 275 @Override 276 public void testFailed(TestIdentifier test, String trace) { 277 super.testFailed(test, trace); 278 // sleep a small amount of time to ensure test failure stack trace makes it into logcat 279 // capture 280 RunUtil.getDefault().sleep(10); 281 InputStreamSource logSource = mDevice.getLogcat(mNumLogcatBytes); 282 super.testLog(String.format("logcat-%s_%s", test.getClassName(), test.getTestName()), 283 LogDataType.TEXT, logSource); 284 logSource.cancel(); 285 } 286 } 287 288 /** 289 * A {@link ResultForwarder} that will forward a screenshot on test failures. 290 */ 291 private static class FailedTestScreenshotGenerator extends ResultForwarder { 292 private ITestDevice mDevice; 293 294 public FailedTestScreenshotGenerator(ITestInvocationListener listener, 295 ITestDevice device) { 296 super(listener); 297 mDevice = device; 298 } 299 300 @Override 301 public void testFailed(TestIdentifier test, String trace) { 302 super.testFailed(test, trace); 303 304 try { 305 InputStreamSource screenSource = mDevice.getScreenshot(); 306 super.testLog(String.format("screenshot-%s_%s", test.getClassName(), 307 test.getTestName()), LogDataType.PNG, screenSource); 308 screenSource.cancel(); 309 } catch (DeviceNotAvailableException e) { 310 // TODO: rethrow this somehow 311 CLog.e("Device %s became unavailable while capturing screenshot, %s", 312 mDevice.getSerialNumber(), e.toString()); 313 } 314 } 315 } 316 317 /** 318 * Create a new {@link CtsTest} that will run the default list of {@link TestPackage}s. 319 */ 320 public CtsTest() { 321 this(0 /*shardAssignment*/, 1 /*totalShards*/); 322 } 323 324 /** 325 * Create a new {@link CtsTest} that will run the given {@link List} of {@link TestPackage}s. 326 */ 327 public CtsTest(int shardAssignment, int totalShards) { 328 if (shardAssignment < 0) { 329 throw new IllegalArgumentException( 330 "shardAssignment cannot be negative. found:" + shardAssignment); 331 } 332 if (totalShards < 1) { 333 throw new IllegalArgumentException( 334 "shardAssignment must be at least 1. found:" + totalShards); 335 } 336 this.mShardAssignment = shardAssignment; 337 this.mTotalShards = totalShards; 338 } 339 340 /** 341 * {@inheritDoc} 342 */ 343 @Override 344 public ITestDevice getDevice() { 345 return mDevice; 346 } 347 348 /** 349 * {@inheritDoc} 350 */ 351 @Override 352 public void setDevice(ITestDevice device) { 353 mDevice = device; 354 } 355 356 /** 357 * Set the plan name to run. 358 * <p/> 359 * Exposed for unit testing 360 */ 361 void setPlanName(String planName) { 362 mPlanName = planName; 363 } 364 365 /** 366 * Set the skip collect device info flag. 367 * <p/> 368 * Exposed for unit testing 369 */ 370 void setSkipDeviceInfo(boolean skipDeviceInfo) { 371 mSkipDeviceInfo = skipDeviceInfo; 372 } 373 374 /** 375 * Adds a package name to the list of test packages to run. 376 * <p/> 377 * Exposed for unit testing 378 */ 379 void addPackageName(String packageName) { 380 mPackageNames.add(packageName); 381 } 382 383 /** 384 * Adds a package name to the list of test packages to exclude. 385 * <p/> 386 * Exposed for unit testing 387 */ 388 void addExcludedPackageName(String packageName) { 389 mExcludedPackageNames.add(packageName); 390 } 391 392 /** 393 * Set the test class name to run. 394 * <p/> 395 * Exposed for unit testing 396 */ 397 void setClassName(String className) { 398 mClassName = className; 399 } 400 401 /** 402 * Set the test method name to run. 403 * <p/> 404 * Exposed for unit testing 405 */ 406 void setMethodName(String methodName) { 407 mMethodName = methodName; 408 } 409 410 /** 411 * Set the test name to run e.g. android.test.cts.SampleTest#testSample 412 * <p/> 413 * Exposed for unit testing 414 */ 415 void setTestName(String testName) { 416 mTestName = testName; 417 } 418 419 /** 420 * Sets the test session id to continue. 421 * <p/> 422 * Exposed for unit testing 423 */ 424 void setContinueSessionId(int sessionId) { 425 mContinueSessionId = sessionId; 426 } 427 428 /** 429 * {@inheritDoc} 430 */ 431 @Override 432 public boolean isResumable() { 433 return mResume; 434 } 435 436 /** 437 * {@inheritDoc} 438 */ 439 @Override 440 public void setBuild(IBuildInfo build) { 441 mCtsBuild = CtsBuildHelper.createBuildHelper(build); 442 mBuildInfo = build; 443 } 444 445 /** 446 * Set the CTS build container. 447 * <p/> 448 * Exposed so unit tests can mock the provided build. 449 */ 450 void setBuildHelper(CtsBuildHelper buildHelper) { 451 mCtsBuild = buildHelper; 452 } 453 454 /** 455 * {@inheritDoc} 456 */ 457 @Override 458 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 459 if (getDevice() == null) { 460 throw new IllegalArgumentException("missing device"); 461 } 462 463 Set<String> abiSet = getAbis(); 464 if (abiSet == null || abiSet.isEmpty()) { 465 throw new IllegalArgumentException("could not get device's ABIs"); 466 } 467 Log.logAndDisplay(LogLevel.INFO, LOG_TAG, "ABIs: " + abiSet); 468 469 checkFields(); 470 setupTestPackageList(abiSet); 471 if (mBugreport) { 472 listener = new FailedTestBugreportGenerator(listener, getDevice()); 473 } 474 if (mScreenshotOnFailures) { 475 listener = new FailedTestScreenshotGenerator(listener, getDevice()); 476 } 477 if (mLogcatOnFailures) { 478 listener = new FailedTestLogcatGenerator(listener, getDevice(), mMaxLogcatBytes); 479 } 480 481 // Setup the a map of Test id to ResultFilter 482 Map<String, ResultFilter> filterMap = new HashMap<>(); 483 int totalTestCount = 0; 484 for (TestPackage testPackage : mTestPackageList) { 485 ResultFilter resultFilter = new ResultFilter(listener, testPackage); 486 totalTestCount += resultFilter.getKnownTestCount(); 487 filterMap.put(testPackage.getPackageDef().getId(), resultFilter); 488 } 489 490 // collect and install the prerequisiteApks first, to save time when multiple test 491 // packages are using the same prerequisite apk 492 Map<String, Set<String>> prerequisiteApks = getPrerequisiteApks(mTestPackageList, abiSet); 493 Collection<String> uninstallPackages = getPrerequisitePackageNames(mTestPackageList); 494 495 try { 496 // always collect the device info, even for resumed runs, since test will likely be 497 // running on a different device 498 collectDeviceInfo(getDevice(), mCtsBuild, listener); 499 preRebootIfNecessary(mTestPackageList); 500 501 mPrevRebootTime = System.currentTimeMillis(); 502 int remainingPackageCount = mTestPackageList.size(); 503 Log.logAndDisplay(LogLevel.INFO, LOG_TAG, 504 String.format("Start test run of %,d packages, containing %,d tests", 505 remainingPackageCount, totalTestCount)); 506 IAbi currentAbi = null; 507 508 for (int i = mLastTestPackageIndex; i < mTestPackageList.size(); i++) { 509 TestPackage testPackage = mTestPackageList.get(i); 510 511 if (currentAbi == null || 512 !currentAbi.getName().equals(testPackage.getAbi().getName())) { 513 currentAbi = testPackage.getAbi(); 514 installPrerequisiteApks( 515 prerequisiteApks.get(currentAbi.getName()), currentAbi); 516 } 517 518 IRemoteTest test = testPackage.getTestForPackage(); 519 if (test instanceof IBuildReceiver) { 520 ((IBuildReceiver) test).setBuild(mBuildInfo); 521 } 522 if (test instanceof IDeviceTest) { 523 ((IDeviceTest) test).setDevice(getDevice()); 524 } 525 if (test instanceof DeqpTestRunner) { 526 ((DeqpTestRunner)test).setCollectLogs(mCollectDeqpLogs); 527 } 528 529 forwardPackageDetails(testPackage.getPackageDef(), listener); 530 test.run(filterMap.get(testPackage.getPackageDef().getId())); 531 if (i < mTestPackageList.size() - 1) { 532 TestPackage nextPackage = mTestPackageList.get(i + 1); 533 rebootIfNecessary(testPackage, nextPackage); 534 changeToHomeScreen(); 535 } 536 // Track of the last complete test package index for resume 537 mLastTestPackageIndex = i; 538 } 539 540 if (mScreenshot) { 541 InputStreamSource screenshotSource = getDevice().getScreenshot(); 542 try { 543 listener.testLog("screenshot", LogDataType.PNG, screenshotSource); 544 } finally { 545 screenshotSource.cancel(); 546 } 547 } 548 549 uninstallPrequisiteApks(uninstallPackages); 550 551 } catch (RuntimeException e) { 552 CLog.e(e); 553 throw e; 554 } catch (Error e) { 555 CLog.e(e); 556 throw e; 557 } finally { 558 for (ResultFilter filter : filterMap.values()) { 559 filter.reportUnexecutedTests(); 560 } 561 } 562 } 563 564 /** 565 * @param allTestPackageDefList The package list to filter 566 * @param deviceAbiSet The ABIs supported by the device being tested 567 * @return A {@link List} of {@link ITestPackageDef}s that should be tested 568 */ 569 private static List<ITestPackageDef> filterByAbi( 570 List<ITestPackageDef> allTestPackageDefList, Set<String> deviceAbiSet) { 571 List<ITestPackageDef> filteredTestPackageDefList = new LinkedList<>(); 572 for (ITestPackageDef testPackageDef : allTestPackageDefList) { 573 if (deviceAbiSet.contains(testPackageDef.getAbi().getName())) { 574 // We only need test packages that are not empty and of matching ABIs 575 filteredTestPackageDefList.add(testPackageDef); 576 } 577 } 578 return filteredTestPackageDefList; 579 } 580 581 /** Reboot then the device iff the list of packages exceeds the minimum */ 582 private void preRebootIfNecessary(List<TestPackage> testPackageList) 583 throws DeviceNotAvailableException { 584 if (mDisableReboot) { 585 return; 586 } 587 588 Set<String> packageNameSet = new HashSet<>(); 589 for (TestPackage testPackage : testPackageList) { 590 // Parse the package name 591 packageNameSet.add(AbiUtils.parseTestName(testPackage.getPackageDef().getId())); 592 } 593 if (packageNameSet.size() < mMinPreRebootPackageCount) { 594 // There is actually only one unique package name. No need to reboot. 595 return; 596 } 597 598 // Reboot is needed 599 Log.logAndDisplay(LogLevel.INFO, LOG_TAG, 600 String.format("Pre-test reboot (%,d packages). Use --disable-reboot to skip", 601 packageNameSet.size())); 602 603 rebootDevice(); 604 } 605 606 private void rebootIfNecessary(TestPackage testFinished, TestPackage testToRun) 607 throws DeviceNotAvailableException { 608 // If there comes spurious failure like INJECT_EVENTS for a package, 609 // reboot it before running it. 610 // Also reboot after package which is know to leave pop-up behind 611 final List<String> rebootAfterList = Arrays.asList( 612 "CtsMediaTestCases", 613 "CtsAccessibilityTestCases"); 614 final List<String> rebootBeforeList = Arrays.asList( 615 "CtsAnimationTestCases", 616 "CtsGraphicsTestCases", 617 "CtsViewTestCases", 618 "CtsWidgetTestCases" ); 619 long intervalInMSec = mRebootIntervalMin * 60 * 1000; 620 if (mDisableReboot || mDevice.getSerialNumber().startsWith("emulator-")) { 621 return; 622 } 623 long currentTime = System.currentTimeMillis(); 624 if (((currentTime - mPrevRebootTime) > intervalInMSec) || 625 rebootAfterList.contains(testFinished.getPackageDef().getName()) || 626 rebootBeforeList.contains(testToRun.getPackageDef().getName()) ) { 627 Log.i(LOG_TAG, 628 String.format("Rebooting after running package %s, before package %s", 629 testFinished.getPackageDef().getName(), 630 testToRun.getPackageDef().getName())); 631 rebootDevice(); 632 mPrevRebootTime = System.currentTimeMillis(); 633 } 634 } 635 636 private void rebootDevice() throws DeviceNotAvailableException { 637 final int TIMEOUT_MS = 10 * 60 * 1000; 638 TestDeviceOptions options = mDevice.getOptions(); 639 // store default value and increase time-out for reboot 640 int rebootTimeout = options.getRebootTimeout(); 641 long onlineTimeout = options.getOnlineTimeout(); 642 options.setRebootTimeout(TIMEOUT_MS); 643 options.setOnlineTimeout(TIMEOUT_MS); 644 mDevice.setOptions(options); 645 646 mDevice.reboot(); 647 648 // restore default values 649 options.setRebootTimeout(rebootTimeout); 650 options.setOnlineTimeout(onlineTimeout); 651 mDevice.setOptions(options); 652 Log.i(LOG_TAG, "Rebooting done"); 653 try { 654 Thread.sleep(mRebootWaitTimeMSec); 655 } catch (InterruptedException e) { 656 Log.i(LOG_TAG, "Boot wait interrupted"); 657 } 658 } 659 660 /** 661 * Remove artifacts like status bar from the previous test. 662 * But this cannot dismiss dialog popped-up. 663 */ 664 private void changeToHomeScreen() throws DeviceNotAvailableException { 665 final String homeCmd = "input keyevent 3"; 666 667 mDevice.executeShellCommand(homeCmd); 668 try { 669 Thread.sleep(1000); 670 } catch (InterruptedException e) { 671 //ignore 672 } 673 } 674 675 /** 676 * Set {@code mTestPackageList} to the list of test packages to run filtered by ABI. 677 */ 678 private void setupTestPackageList(Set<String> abis) throws DeviceNotAvailableException { 679 if (!mTestPackageList.isEmpty()) { 680 Log.logAndDisplay(LogLevel.INFO, LOG_TAG, "Resume tests using existing package list"); 681 return; 682 } 683 try { 684 // Collect ALL tests 685 ITestPackageRepo testRepo = createTestCaseRepo(); 686 List<ITestPackageDef> testPkgDefs = new ArrayList<>(getAvailableTestPackages(testRepo)); 687 testPkgDefs = filterByAbi(testPkgDefs, abis); 688 // Note: run() relies on the fact that the list is reliably sorted for sharding purposes 689 Collections.sort(testPkgDefs); 690 // Create test package list. 691 List<TestPackage> testPackageList = new ArrayList<>(); 692 for (ITestPackageDef testPackageDef : testPkgDefs) { 693 // Note: createTest filters the test list inside of testPackageDef by exclusion list 694 IRemoteTest testForPackage = testPackageDef.createTest(mCtsBuild.getTestCasesDir()); 695 if (testPackageDef.getTests().size() > 0) { 696 testPackageList.add(new TestPackage(testPackageDef, testForPackage)); 697 } 698 } 699 700 // Filter by shard 701 int numTestPackages = testPackageList.size(); 702 int totalShards = Math.min(mTotalShards, numTestPackages); 703 704 List<TestPackage> shardTestPackageList = new ArrayList<>(); 705 for (int i = mShardAssignment; i < numTestPackages; i += totalShards) { 706 shardTestPackageList.add(testPackageList.get(i)); 707 } 708 mTestPackageList.addAll(shardTestPackageList); 709 } catch (FileNotFoundException e) { 710 throw new IllegalArgumentException("failed to find test plan file", e); 711 } catch (ParseException e) { 712 throw new IllegalArgumentException("failed to parse test plan file", e); 713 } catch (ConfigurationException e) { 714 throw new IllegalArgumentException("failed to process arguments", e); 715 } 716 } 717 718 /** 719 * Return the {@link Set} of {@link ITestPackageDef}s to run unfiltered by ABI 720 * 721 * @return the {@link Set} of {@link ITestPackageDef}s to run 722 * @throws ParseException 723 * @throws FileNotFoundException 724 * @throws ConfigurationException 725 */ 726 private Set<ITestPackageDef> getAvailableTestPackages(ITestPackageRepo testRepo) 727 throws ParseException, FileNotFoundException, ConfigurationException { 728 // use LinkedHashSet to have predictable iteration order 729 Set<ITestPackageDef> testPkgDefs = new LinkedHashSet<>(); 730 if (mPlanName != null) { 731 Log.i(LOG_TAG, String.format("Executing CTS test plan %s", mPlanName)); 732 File ctsPlanFile = mCtsBuild.getTestPlanFile(mPlanName); 733 ITestPlan plan = createPlan(mPlanName); 734 plan.parse(createXmlStream(ctsPlanFile)); 735 736 for (String testId : plan.getTestIds()) { 737 if (mExcludedPackageNames.contains(AbiUtils.parseTestName(testId))) { 738 continue; 739 } 740 ITestPackageDef testPackageDef = testRepo.getTestPackage(testId); 741 if (testPackageDef == null) { 742 CLog.e("Could not find test id %s referenced in plan %s", testId, mPlanName); 743 continue; 744 } 745 746 testPackageDef.setTestFilter(plan.getTestFilter(testId)); 747 testPkgDefs.add(testPackageDef); 748 } 749 } else if (mPackageNames.size() > 0){ 750 Log.i(LOG_TAG, String.format("Executing test packages %s", mPackageNames)); 751 752 Map<String, List<ITestPackageDef>> testPackageDefMap = 753 testRepo.getTestPackageDefsByName(); 754 755 for (String name : mPackageNames) { 756 if (!testPackageDefMap.containsKey(name)) { 757 throw new IllegalArgumentException(String.format( 758 "Could not find test package %s. " + 759 "Use 'list packages' to see available packages.", name)); 760 } 761 testPkgDefs.addAll(testPackageDefMap.get(name)); 762 } 763 } else if (mClassName != null) { 764 Log.i(LOG_TAG, String.format("Executing CTS test class %s", mClassName)); 765 testPkgDefs.addAll(buildTestPackageDefSet(testRepo, mClassName, mMethodName)); 766 } else if (mTestName != null) { 767 Log.i(LOG_TAG, String.format("Executing CTS test %s", mTestName)); 768 String [] split = mTestName.split("#"); 769 if (split.length != 2) { 770 Log.logAndDisplay(LogLevel.WARN, LOG_TAG, String.format( 771 "Could not parse class and method from test %s", mTestName)); 772 } else { 773 String className = split[0]; 774 String methodName = split[1]; 775 testPkgDefs.addAll(buildTestPackageDefSet(testRepo, className, methodName)); 776 } 777 } else if (mContinueSessionId != null) { 778 // create an in-memory derived plan that contains the notExecuted tests from previous 779 // session use timestamp as plan name so it will hopefully be unique 780 String uniquePlanName = Long.toString(System.currentTimeMillis()); 781 PlanCreator planCreator = new PlanCreator(uniquePlanName, mContinueSessionId, 782 CtsTestStatus.NOT_EXECUTED); 783 ITestPlan plan = createPlan(planCreator); 784 for (String testId : plan.getTestIds()) { 785 if (mExcludedPackageNames.contains(AbiUtils.parseTestName(testId))) { 786 continue; 787 } 788 ITestPackageDef testPackageDef = testRepo.getTestPackage(testId); 789 if (testPackageDef == null) { 790 CLog.e("Could not find test id %s referenced in plan %s", testId, mPlanName); 791 continue; 792 } 793 794 testPackageDef.setTestFilter(plan.getTestFilter(testId)); 795 testPkgDefs.add(testPackageDef); 796 } 797 } else { 798 // should never get here - was checkFields() not called? 799 throw new IllegalStateException("nothing to run?"); 800 } 801 return testPkgDefs; 802 } 803 804 /** 805 * Return the list of unique prerequisite Android package names 806 * 807 * @param testPackages The {@link TestPackage}s that contain prerequisites 808 */ 809 private Collection<String> getPrerequisitePackageNames(List<TestPackage> testPackages) { 810 Set<String> pkgNames = new HashSet<>(); 811 for (TestPackage testPkg : testPackages) { 812 String pkgName = testPkg.mPackageDef.getTargetPackageName(); 813 if (pkgName != null) { 814 pkgNames.add(pkgName); 815 } 816 } 817 return pkgNames; 818 } 819 820 /** 821 * @return a {@link Set} containing {@link ITestPackageDef}s pertaining to the given 822 * {@code className} and {@code methodName}. 823 */ 824 private static Set<ITestPackageDef> buildTestPackageDefSet( 825 ITestPackageRepo testRepo, String className, String methodName) { 826 Set<ITestPackageDef> testPkgDefs = new LinkedHashSet<>(); 827 // try to find packages to run from class name 828 List<String> packageIds = testRepo.findPackageIdsForTest(className); 829 if (packageIds.isEmpty()) { 830 Log.logAndDisplay(LogLevel.WARN, LOG_TAG, String.format( 831 "Could not find package for test class %s", className)); 832 } 833 for (String packageId: packageIds) { 834 ITestPackageDef testPackageDef = testRepo.getTestPackage(packageId); 835 if (testPackageDef != null) { 836 testPackageDef.setClassName(className, methodName); 837 testPkgDefs.add(testPackageDef); 838 } 839 } 840 return testPkgDefs; 841 } 842 843 /** 844 * Return the list (by abi) of unique prerequisite apks to install 845 * 846 * @param testPackages The {@link List} of {@link TestPackage} that contain prerequisite APKs 847 */ 848 private Map<String, Set<String>> getPrerequisiteApks( 849 List<TestPackage> testPackages, Set<String> abiSet) { 850 Map<String, Set<String>> abiToApkMap = new HashMap<>(); 851 for (TestPackage testPkg : testPackages) { 852 if (testPkg.getKnownTests().size() == 0) { 853 // No tests, no point in installing pre-reqs 854 continue; 855 } 856 String apkName = testPkg.mPackageDef.getTargetApkName(); 857 if (apkName == null) { 858 continue; 859 } 860 String abiName = testPkg.getAbi().getName(); 861 if (!abiSet.contains(abiName)) { 862 continue; 863 } 864 865 if (!abiToApkMap.containsKey(abiName)) { 866 abiToApkMap.put(abiName, new HashSet<String>()); 867 } 868 abiToApkMap.get(abiName).add(apkName); 869 } 870 return abiToApkMap; 871 } 872 873 /** 874 * FIXME eventually this should be removed once we get rid of CtsTestStubs, any other 875 * prerequisite apks should be installed by the test runner 876 * 877 * Install the collection of test apk file names 878 * 879 * @param prerequisiteApks The APKs that must be installed 880 * @throws DeviceNotAvailableException 881 */ 882 private void installPrerequisiteApks(Collection<String> prerequisiteApks, IAbi abi) 883 throws DeviceNotAvailableException { 884 if (prerequisiteApks == null) { 885 return; 886 } 887 Log.logAndDisplay(LogLevel.INFO, LOG_TAG, "Installing prerequisites"); 888 for (String apkName : prerequisiteApks) { 889 try { 890 File apkFile = mCtsBuild.getTestApp(apkName); 891 String[] options = {AbiUtils.createAbiFlag(abi.getName())}; 892 String errorCode = getDevice().installPackage(apkFile, true, options); 893 if (errorCode != null) { 894 CLog.e("Failed to install %s. Reason: %s", apkName, errorCode); 895 } 896 } catch (FileNotFoundException e) { 897 CLog.e("Could not find test apk %s", apkName); 898 } 899 } 900 } 901 902 /** 903 * Uninstalls the collection of android package names from device. 904 * 905 * @param uninstallPackages The packages that must be uninstalled 906 */ 907 private void uninstallPrequisiteApks(Collection<String> uninstallPackages) 908 throws DeviceNotAvailableException { 909 for (String pkgName : uninstallPackages) { 910 getDevice().uninstallPackage(pkgName); 911 } 912 } 913 914 /** 915 * {@inheritDoc} 916 */ 917 @Override 918 public Collection<IRemoteTest> split() { 919 if (mShards <= 1) { 920 return null; 921 } 922 checkFields(); 923 924 List<IRemoteTest> shardQueue = new LinkedList<>(); 925 for (int shardAssignment = 0; shardAssignment < mShards; shardAssignment++) { 926 CtsTest ctsTest = new CtsTest(shardAssignment, mShards /* totalShards */); 927 OptionCopier.copyOptionsNoThrow(this, ctsTest); 928 // Set the shard count because the copy option on the previous line copies 929 // over the mShard value 930 ctsTest.mShards = 0; 931 shardQueue.add(ctsTest); 932 } 933 934 return shardQueue; 935 } 936 937 /** 938 * Runs the device info collector instrumentation on device, and forwards it to test listeners 939 * as run metrics. 940 * <p/> 941 * Exposed so unit tests can mock. 942 * 943 * @throws DeviceNotAvailableException 944 */ 945 void collectDeviceInfo(ITestDevice device, CtsBuildHelper ctsBuild, 946 ITestInvocationListener listener) throws DeviceNotAvailableException { 947 if (!mSkipDeviceInfo) { 948 String abi = AbiFormatter.getDefaultAbi(device, ""); 949 DeviceInfoCollector.collectDeviceInfo(device, abi, ctsBuild.getTestCasesDir(), listener); 950 } 951 } 952 953 /** 954 * Factory method for creating a {@link ITestPackageRepo}. 955 * <p/> 956 * Exposed for unit testing 957 */ 958 ITestPackageRepo createTestCaseRepo() { 959 return new TestPackageRepo(mCtsBuild.getTestCasesDir(), mIncludeKnownFailures); 960 } 961 962 /** 963 * Factory method for creating a {@link TestPlan}. 964 * <p/> 965 * Exposed for unit testing 966 */ 967 ITestPlan createPlan(String planName) { 968 return new TestPlan(planName, AbiUtils.getAbisSupportedByCts()); 969 } 970 971 /** 972 * Gets the set of ABIs supported by both CTS and the device under test 973 * <p/> 974 * Exposed for unit testing 975 * @return The set of ABIs to run the tests on 976 * @throws DeviceNotAvailableException 977 */ 978 Set<String> getAbis() throws DeviceNotAvailableException { 979 String bitness = (mForceAbi == null) ? "" : mForceAbi; 980 Set<String> abis = new HashSet<>(); 981 for (String abi : AbiFormatter.getSupportedAbis(mDevice, bitness)) { 982 if (AbiUtils.isAbiSupportedByCts(abi)) { 983 abis.add(abi); 984 } 985 } 986 return abis; 987 } 988 989 /** 990 * Factory method for creating a {@link TestPlan} from a {@link PlanCreator}. 991 * <p/> 992 * Exposed for unit testing 993 * @throws ConfigurationException 994 */ 995 ITestPlan createPlan(PlanCreator planCreator) 996 throws ConfigurationException { 997 return planCreator.createDerivedPlan(mCtsBuild, AbiUtils.getAbisSupportedByCts()); 998 } 999 1000 /** 1001 * Factory method for creating a {@link InputStream} from a plan xml file. 1002 * <p/> 1003 * Exposed for unit testing 1004 */ 1005 InputStream createXmlStream(File xmlFile) throws FileNotFoundException { 1006 return new BufferedInputStream(new FileInputStream(xmlFile)); 1007 } 1008 1009 private void checkFields() { 1010 // for simplicity of command line usage, make --plan, --package, --test and --class mutually 1011 // exclusive 1012 boolean mutualExclusiveArgs = xor(mPlanName != null, mPackageNames.size() > 0, 1013 mClassName != null, mContinueSessionId != null, mTestName != null); 1014 1015 if (!mutualExclusiveArgs) { 1016 throw new IllegalArgumentException(String.format( 1017 "Ambiguous or missing arguments. " + 1018 "One and only one of --%s --%s(s), --%s or --%s to run can be specified", 1019 PLAN_OPTION, PACKAGE_OPTION, CLASS_OPTION, CONTINUE_OPTION)); 1020 } 1021 if (mMethodName != null && mClassName == null) { 1022 throw new IllegalArgumentException(String.format( 1023 "Must specify --%s when --%s is used", CLASS_OPTION, METHOD_OPTION)); 1024 } 1025 if (mCtsBuild == null) { 1026 throw new IllegalArgumentException("missing CTS build"); 1027 } 1028 } 1029 1030 /** 1031 * Helper method to perform exclusive or on list of boolean arguments 1032 * 1033 * @param args set of booleans on which to perform exclusive or 1034 * @return <code>true</code> if one and only one of <var>args</code> is <code>true</code>. 1035 * Otherwise return <code>false</code>. 1036 */ 1037 private static boolean xor(boolean... args) { 1038 boolean currentVal = args[0]; 1039 for (int i=1; i < args.length; i++) { 1040 if (currentVal && args[i]) { 1041 return false; 1042 } 1043 currentVal |= args[i]; 1044 } 1045 return currentVal; 1046 } 1047 1048 /** 1049 * Forward the digest and package name to the listener as a metric 1050 * 1051 * @param listener Handles test results 1052 */ 1053 private static void forwardPackageDetails(ITestPackageDef def, ITestInvocationListener listener) { 1054 Map<String, String> metrics = new HashMap<>(3); 1055 metrics.put(PACKAGE_NAME_METRIC, def.getName()); 1056 metrics.put(PACKAGE_ABI_METRIC, def.getAbi().getName()); 1057 metrics.put(PACKAGE_DIGEST_METRIC, def.getDigest()); 1058 listener.testRunStarted(def.getId(), 0); 1059 listener.testRunEnded(0, metrics); 1060 } 1061 } 1062
