1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.support.test.aupt; 18 19 import android.app.Service; 20 import android.content.Context; 21 import android.content.ContextWrapper; 22 import android.content.Intent; 23 import android.content.IntentFilter; 24 import android.os.Bundle; 25 import android.os.Environment; 26 import android.os.IBinder; 27 import android.os.SystemClock; 28 import android.support.test.uiautomator.UiDevice; 29 import android.test.AndroidTestRunner; 30 import android.test.InstrumentationTestCase; 31 import android.test.InstrumentationTestRunner; 32 import android.util.Log; 33 34 import junit.framework.AssertionFailedError; 35 import junit.framework.Test; 36 import junit.framework.TestCase; 37 import junit.framework.TestListener; 38 import junit.framework.TestResult; 39 import junit.framework.TestSuite; 40 41 import java.io.BufferedReader; 42 import java.io.File; 43 import java.io.FileInputStream; 44 import java.io.IOException; 45 import java.io.InputStreamReader; 46 import java.text.SimpleDateFormat; 47 import java.util.ArrayList; 48 import java.util.Arrays; 49 import java.util.Comparator; 50 import java.util.Date; 51 import java.util.HashMap; 52 import java.util.List; 53 import java.util.Map; 54 import java.util.Random; 55 import java.util.concurrent.TimeUnit; 56 import java.util.concurrent.TimeoutException; 57 58 /** 59 * Ultra-fancy TestRunner to use when running AUPT: supports 60 * 61 * - Picking up tests from dexed JARs 62 * - Running tests for multiple iterations or in a custom order 63 * - Terminating tests after UI errors, timeouts, or when dependent processes die 64 * - Injecting additional information into custom TestCase subclasses 65 * - Passing through continuous metric-collection to a DataCollector instance 66 * - Collecting bugreports and heapdumps 67 * 68 */ 69 public class AuptTestRunner extends InstrumentationTestRunner { 70 /* Constants */ 71 private static final String LOG_TAG = AuptTestRunner.class.getSimpleName(); 72 private static final Long ANR_DELAY = 30000L; 73 private static final Long DEFAULT_SUITE_TIMEOUT = 0L; 74 private static final Long DEFAULT_TEST_TIMEOUT = 10L; 75 private static final SimpleDateFormat SCREENSHOT_DATE_FORMAT = 76 new SimpleDateFormat("dd-mm-yy:HH:mm:ss:SSS"); 77 78 /* Keep a pointer to our argument bundle around for testing */ 79 private Bundle mParams; 80 81 /* Primitive Parameters */ 82 private boolean mDeleteOldFiles; 83 private long mFileRetainCount; 84 private boolean mGenerateAnr; 85 private boolean mRecordMeminfo; 86 private long mIterations; 87 private long mSeed; 88 89 /* Dumpheap Parameters */ 90 private boolean mDumpheapEnabled; 91 private long mDumpheapInterval; 92 private long mDumpheapThreshold; 93 private long mMaxDumpheaps; 94 95 /* String Parameters */ 96 private List<String> mJars = new ArrayList<>(); 97 private List<String> mMemoryTrackedProcesses = new ArrayList<>(); 98 private List<String> mFinishCommands; 99 100 /* Other Parameters */ 101 private File mResultsDirectory; 102 103 /* Helpers */ 104 private Scheduler mScheduler; 105 private DataCollector mDataCollector; 106 private DexTestRunner mRunner; 107 108 /* Logging */ 109 private ProcessStatusTracker mProcessTracker; 110 private List<MemHealthRecord> mMemHealthRecords = new ArrayList<>(); 111 private Map<String, Long> mDumpheapCount = new HashMap<>(); 112 private Map<String, Long> mLastDumpheap = new HashMap<>(); 113 114 /* Test Initialization */ 115 @Override 116 public void onCreate(Bundle params) { 117 mParams = params; 118 119 // Parse out primitive parameters 120 mIterations = parseLongParam("iterations", 1); 121 mRecordMeminfo = parseBoolParam("record_meminfo", false); 122 mDumpheapEnabled = parseBoolParam("enableDumpheap", false); 123 mDumpheapThreshold = parseLongParam("dumpheapThreshold", 200 * 1024 * 1024); 124 mDumpheapInterval = parseLongParam("dumpheapInterval", 60 * 60 * 1000); 125 mMaxDumpheaps = parseLongParam("maxDumpheaps", 5); 126 mSeed = parseLongParam("seed", new Random().nextLong()); 127 128 // Option: -e finishCommand 'a;b;c;d' 129 String finishCommandArg = parseStringParam("finishCommand", null); 130 mFinishCommands = 131 finishCommandArg == null 132 ? Arrays.<String>asList() 133 : Arrays.asList(finishCommandArg.split("\\s*;\\s*")); 134 135 // Option: -e shuffle true 136 mScheduler = parseBoolParam("shuffle", false) 137 ? Scheduler.shuffled(new Random(mSeed), mIterations) 138 : Scheduler.sequential(mIterations); 139 140 // Option: -e jars aupt-app-tests.jar:... 141 mJars.addAll(DexTestRunner.parseDexedJarPaths(parseStringParam("jars", ""))); 142 143 // Option: -e trackMemory com.pkg1,com.pkg2,... 144 String memoryTrackedProcesses = parseStringParam("trackMemory", null); 145 146 if (memoryTrackedProcesses != null) { 147 mMemoryTrackedProcesses = Arrays.asList(memoryTrackedProcesses.split(",")); 148 } else { 149 try { 150 // Deprecated approach: get tracked processes from a file. 151 String trackMemoryFileName = 152 Environment.getExternalStorageDirectory() + "/track_memory.txt"; 153 154 BufferedReader reader = new BufferedReader(new InputStreamReader( 155 new FileInputStream(new File(trackMemoryFileName)))); 156 157 mMemoryTrackedProcesses = Arrays.asList(reader.readLine().split(",")); 158 reader.close(); 159 } catch (NullPointerException | IOException ex) { 160 mMemoryTrackedProcesses = Arrays.asList(); 161 } 162 } 163 164 // Option: -e detectKill com.pkg1,...,com.pkg8 165 String processes = parseStringParam("detectKill", null); 166 167 if (processes != null) { 168 mProcessTracker = new ProcessStatusTracker(processes.split(",")); 169 } else { 170 mProcessTracker = new ProcessStatusTracker(new String[] {}); 171 } 172 173 // Option: -e outputLocation aupt_results 174 mResultsDirectory = new File(Environment.getExternalStorageDirectory(), 175 parseStringParam("outputLocation", "aupt_results")); 176 if (!mResultsDirectory.exists() && !mResultsDirectory.mkdirs()) { 177 Log.w(LOG_TAG, "Could not find or create output directory " + mResultsDirectory); 178 } 179 180 // Option: -e fileRetainCount 1 181 mFileRetainCount = parseLongParam("fileRetainCount", -1); 182 mDeleteOldFiles = (mFileRetainCount != -1); 183 184 // Primary logging infrastructure 185 mDataCollector = new DataCollector( 186 TimeUnit.MINUTES.toMillis(parseLongParam("bugreportInterval", 0)), 187 TimeUnit.MINUTES.toMillis(parseLongParam("jankInterval", 0)), 188 TimeUnit.MINUTES.toMillis(parseLongParam("meminfoInterval", 0)), 189 TimeUnit.MINUTES.toMillis(parseLongParam("cpuinfoInterval", 0)), 190 TimeUnit.MINUTES.toMillis(parseLongParam("fragmentationInterval", 0)), 191 TimeUnit.MINUTES.toMillis(parseLongParam("ionInterval", 0)), 192 TimeUnit.MINUTES.toMillis(parseLongParam("pagetypeinfoInterval", 0)), 193 TimeUnit.MINUTES.toMillis(parseLongParam("traceInterval", 0)), 194 TimeUnit.MINUTES.toMillis(parseLongParam("bugreportzInterval", 0)), 195 mResultsDirectory, this); 196 197 // Make our TestRunner and make sure we injectInstrumentation. 198 mRunner = new DexTestRunner(this, mScheduler, mJars, 199 TimeUnit.MINUTES.toMillis(parseLongParam("testCaseTimeout", DEFAULT_TEST_TIMEOUT)), 200 TimeUnit.MINUTES.toMillis(parseLongParam("suiteTimeout", DEFAULT_SUITE_TIMEOUT))) { 201 @Override 202 public void runTest(TestResult result) { 203 for (TestCase test: mTestCases) { 204 injectInstrumentation(test); 205 } 206 207 try { 208 super.runTest(result); 209 } finally { 210 mDataCollector.stop(); 211 } 212 } 213 }; 214 215 // Aupt's TestListeners 216 mRunner.addTestListener(new PeriodicHeapDumper()); 217 mRunner.addTestListener(new MemHealthRecorder()); 218 mRunner.addTestListener(new DcimCleaner()); 219 mRunner.addTestListener(new PidChecker()); 220 mRunner.addTestListener(new TimeoutStackDumper()); 221 mRunner.addTestListener(new MemInfoDumper()); 222 mRunner.addTestListener(new FinishCommandRunner()); 223 mRunner.addTestListenerIf(parseBoolParam("generateANR", false), new ANRTrigger()); 224 mRunner.addTestListenerIf(parseBoolParam("quitOnError", false), new QuitOnErrorListener()); 225 mRunner.addTestListenerIf(parseBoolParam("checkBattery", false), new BatteryChecker()); 226 mRunner.addTestListenerIf(parseBoolParam("screenshots", false), new Screenshotter()); 227 228 // Start our loggers 229 mDataCollector.start(); 230 231 // Start the test 232 super.onCreate(params); 233 } 234 235 /* Option-parsing helpers */ 236 237 private long parseLongParam(String key, long alternative) throws NumberFormatException { 238 if (mParams.containsKey(key)) { 239 return Long.parseLong(mParams.getString(key)); 240 } else { 241 return alternative; 242 } 243 } 244 245 private boolean parseBoolParam(String key, boolean alternative) 246 throws NumberFormatException { 247 if (mParams.containsKey(key)) { 248 return Boolean.parseBoolean(mParams.getString(key)); 249 } else { 250 return alternative; 251 } 252 } 253 254 private String parseStringParam(String key, String alternative) { 255 if (mParams.containsKey(key)) { 256 return mParams.getString(key); 257 } else { 258 return alternative; 259 } 260 } 261 262 /* Utility methods */ 263 264 /** 265 * Injects instrumentation into InstrumentationTestCase and AuptTestCase instances 266 */ 267 private void injectInstrumentation(Test test) { 268 if (InstrumentationTestCase.class.isAssignableFrom(test.getClass())) { 269 InstrumentationTestCase instrTest = (InstrumentationTestCase) test; 270 271 instrTest.injectInstrumentation(AuptTestRunner.this); 272 } 273 } 274 275 /* Passthrough to our DexTestRunner */ 276 @Override 277 protected AndroidTestRunner getAndroidTestRunner() { 278 return mRunner; 279 } 280 281 @Override 282 public Context getTargetContext() { 283 return new ContextWrapper(super.getTargetContext()) { 284 @Override 285 public ClassLoader getClassLoader() { 286 if(mRunner != null) { 287 return mRunner.getDexClassLoader(); 288 } else { 289 throw new RuntimeException("DexTestRunner not initialized!"); 290 } 291 } 292 }; 293 } 294 295 /** 296 * A simple abstract instantiation of TestListener 297 * 298 * Primarily meant to work around Java 7's lack of interface-default methods. 299 */ 300 abstract static class AuptListener implements TestListener { 301 /** Called when a test throws an exception. */ 302 public void addError(Test test, Throwable t) {} 303 304 /** Called when a test fails. */ 305 public void addFailure(Test test, AssertionFailedError t) {} 306 307 /** Called whenever a test ends. */ 308 public void endTest(Test test) {} 309 310 /** Called whenever a test begins. */ 311 public void startTest(Test test) {} 312 } 313 314 /** 315 * Periodically Heap-dump to assist with memory-leaks. 316 */ 317 private class PeriodicHeapDumper extends AuptListener { 318 private Thread mHeapDumpThread; 319 320 private class InternalHeapDumper implements Runnable { 321 private void recordDumpheap(String proc, long pss) throws IOException { 322 if (!mDumpheapEnabled) { 323 return; 324 } 325 Long count = mDumpheapCount.get(proc); 326 if (count == null) { 327 count = 0L; 328 } 329 Long lastDumpheap = mLastDumpheap.get(proc); 330 if (lastDumpheap == null) { 331 lastDumpheap = 0L; 332 } 333 long currentTime = SystemClock.uptimeMillis(); 334 if (pss > mDumpheapThreshold && count < mMaxDumpheaps && 335 currentTime - lastDumpheap > mDumpheapInterval) { 336 recordDumpheap(proc); 337 mDumpheapCount.put(proc, count + 1); 338 mLastDumpheap.put(proc, currentTime); 339 } 340 } 341 342 private void recordDumpheap(String proc) throws IOException { 343 long count = mDumpheapCount.get(proc); 344 345 String filename = String.format("dumpheap-%s-%d", proc, count); 346 String tempFilename = "/data/local/tmp/" + filename; 347 String finalFilename = mResultsDirectory + "/" + filename; 348 349 AuptTestRunner.this.getUiAutomation().executeShellCommand( 350 String.format("am dumpheap %s %s", proc, tempFilename)); 351 352 SystemClock.sleep(3000); 353 354 AuptTestRunner.this.getUiAutomation().executeShellCommand( 355 String.format("cp %s %s", tempFilename, finalFilename)); 356 } 357 358 public void run() { 359 try { 360 while (true) { 361 Thread.sleep(mDumpheapInterval); 362 363 for(String proc : mMemoryTrackedProcesses) { 364 recordDumpheap(proc); 365 } 366 } 367 } catch (InterruptedException iex) { 368 } catch (IOException ioex) { 369 Log.e(LOG_TAG, "Failed to write heap dump!", ioex); 370 } 371 } 372 } 373 374 @Override 375 public void startTest(Test test) { 376 mHeapDumpThread = new Thread(new InternalHeapDumper()); 377 mHeapDumpThread.start(); 378 } 379 380 @Override 381 public void endTest(Test test) { 382 try { 383 mHeapDumpThread.interrupt(); 384 mHeapDumpThread.join(); 385 } catch (InterruptedException iex) { } 386 } 387 } 388 389 /** 390 * Dump memory info on test start/stop 391 */ 392 private class MemInfoDumper extends AuptListener { 393 private void dumpMemInfo() { 394 if (mRecordMeminfo) { 395 FilesystemUtil.dumpMeminfo(AuptTestRunner.this, "MemInfoDumper"); 396 } 397 } 398 399 @Override 400 public void startTest(Test test) { 401 dumpMemInfo(); 402 } 403 404 @Override 405 public void endTest(Test test) { 406 dumpMemInfo(); 407 } 408 } 409 410 /** 411 * Record all of our MemHealthRecords 412 */ 413 private class MemHealthRecorder extends AuptListener { 414 @Override 415 public void startTest(Test test) { 416 recordMemHealth(); 417 } 418 419 @Override 420 public void endTest(Test test) { 421 recordMemHealth(); 422 423 try { 424 MemHealthRecord.saveVerbose(mMemHealthRecords, 425 new File(mResultsDirectory, "memory-health.txt").getPath()); 426 MemHealthRecord.saveCsv(mMemHealthRecords, 427 new File(mResultsDirectory, "memory-health-details.txt").getPath()); 428 429 mMemHealthRecords.clear(); 430 } catch (IOException ioex) { 431 Log.e(LOG_TAG, "Error writing MemHealthRecords", ioex); 432 } 433 } 434 435 private void recordMemHealth() { 436 try { 437 mMemHealthRecords.addAll(MemHealthRecord.get( 438 AuptTestRunner.this, 439 mMemoryTrackedProcesses, 440 System.currentTimeMillis(), 441 getForegroundProcs())); 442 } catch (IOException ioex) { 443 Log.e(LOG_TAG, "Error collecting MemHealthRecords", ioex); 444 } 445 } 446 447 private List<String> getForegroundProcs() { 448 List<String> foregroundProcs = new ArrayList<String>(); 449 try { 450 String compactMeminfo = MemHealthRecord.getProcessOutput(AuptTestRunner.this, 451 "dumpsys meminfo -c"); 452 453 for (String line : compactMeminfo.split("\\r?\\n")) { 454 if (line.contains("proc,fore")) { 455 String proc = line.split(",")[2]; 456 foregroundProcs.add(proc); 457 } 458 } 459 } catch (IOException e) { 460 Log.e(LOG_TAG, "Error while getting foreground process", e); 461 } finally { 462 return foregroundProcs; 463 } 464 } 465 } 466 467 /** 468 * Kills application and dumps UI Hierarchy on test error 469 */ 470 private class QuitOnErrorListener extends AuptListener { 471 @Override 472 public void addError(Test test, Throwable t) { 473 Log.e(LOG_TAG, "Caught exception from a test", t); 474 475 if ((t instanceof AuptTerminator)) { 476 throw (AuptTerminator)t; 477 } else { 478 479 // Check if our exception is caused by process dependency 480 if (test instanceof AuptTestCase) { 481 mProcessTracker.setUiAutomation(getUiAutomation()); 482 mProcessTracker.verifyRunningProcess(); 483 } 484 485 // If that didn't throw, then dump our hierarchy 486 Log.v(LOG_TAG, "Dumping UI hierarchy"); 487 try { 488 UiDevice.getInstance(AuptTestRunner.this).dumpWindowHierarchy( 489 new File("/data/local/tmp/error_dump.xml")); 490 } catch (IOException e) { 491 Log.w(LOG_TAG, "Failed to create UI hierarchy dump for UI error", e); 492 } 493 } 494 495 // Quit on an error 496 throw new AuptTerminator(t.getMessage(), t); 497 } 498 499 @Override 500 public void addFailure(Test test, AssertionFailedError t) { 501 // Quit on an error 502 throw new AuptTerminator(t.getMessage(), t); 503 } 504 } 505 506 /** 507 * Makes sure the processes this test requires are all alive 508 */ 509 private class PidChecker extends AuptListener { 510 @Override 511 public void startTest(Test test) { 512 mProcessTracker.setUiAutomation(getUiAutomation()); 513 mProcessTracker.verifyRunningProcess(); 514 } 515 516 @Override 517 public void endTest(Test test) { 518 mProcessTracker.verifyRunningProcess(); 519 } 520 } 521 522 /** 523 * Initialization for tests that touch the camera 524 */ 525 private class DcimCleaner extends AuptListener { 526 @Override 527 public void startTest(Test test) { 528 if (!mDeleteOldFiles) { 529 return; 530 } 531 532 File dcimFolder = new File(Environment.getExternalStorageDirectory(), "DCIM"); 533 File cameraFolder = new File(dcimFolder, "Camera"); 534 535 if (dcimFolder.exists()) { 536 if (cameraFolder.exists()) { 537 File[] allMediaFiles = cameraFolder.listFiles(); 538 Arrays.sort(allMediaFiles, new Comparator<File>() { 539 public int compare(File f1, File f2) { 540 return Long.valueOf(f1.lastModified()).compareTo(f2.lastModified()); 541 } 542 }); 543 for (int i = 0; i < allMediaFiles.length - mFileRetainCount; i++) { 544 allMediaFiles[i].delete(); 545 } 546 } else { 547 Log.w(LOG_TAG, "No Camera folder found to delete from."); 548 } 549 } else { 550 Log.w(LOG_TAG, "No DCIM folder found to delete from."); 551 } 552 } 553 } 554 555 /** 556 * Makes sure the battery hasn't died before and after each test. 557 */ 558 private class BatteryChecker extends AuptListener { 559 private static final double BATTERY_THRESHOLD = 0.05; 560 561 private void checkBattery() { 562 Intent batteryIntent = getContext().registerReceiver(null, 563 new IntentFilter(Intent.ACTION_BATTERY_CHANGED)); 564 int rawLevel = batteryIntent.getIntExtra("level", -1); 565 int scale = batteryIntent.getIntExtra("scale", -1); 566 567 if (rawLevel < 0 || scale <= 0) { 568 return; 569 } 570 571 double level = (double) rawLevel / (double) scale; 572 if (level < BATTERY_THRESHOLD) { 573 throw new AuptTerminator(String.format("Current battery level %f lower than %f", 574 level, 575 BATTERY_THRESHOLD)); 576 } 577 } 578 579 @Override 580 public void startTest(Test test) { 581 checkBattery(); 582 } 583 } 584 585 /** 586 * Generates heap dumps when a test times out 587 */ 588 private class TimeoutStackDumper extends AuptListener { 589 private String getStackTraces() { 590 StringBuilder sb = new StringBuilder(); 591 Map<Thread, StackTraceElement[]> stacks = Thread.getAllStackTraces(); 592 for (Thread t : stacks.keySet()) { 593 sb.append(t.toString()).append('\n'); 594 for (StackTraceElement ste : t.getStackTrace()) { 595 sb.append("\tat ").append(ste.toString()).append('\n'); 596 } 597 sb.append('\n'); 598 } 599 return sb.toString(); 600 } 601 602 @Override 603 public void addError(Test test, Throwable t) { 604 if (t instanceof TimeoutException) { 605 Log.d("THREAD_DUMP", getStackTraces()); 606 } 607 } 608 } 609 610 /** Generates ANRs when a test takes too long. */ 611 private class ANRTrigger extends AuptListener { 612 @Override 613 public void addError(Test test, Throwable t) { 614 if (t instanceof TimeoutException) { 615 Context ctx = getTargetContext(); 616 Log.d(LOG_TAG, "About to induce artificial ANR for debugging"); 617 ctx.startService(new Intent(ctx, AnrGenerator.class)); 618 619 try { 620 Thread.sleep(ANR_DELAY); 621 } catch (InterruptedException e) { 622 throw new RuntimeException("Interrupted while waiting for AnrGenerator..."); 623 } 624 } 625 } 626 627 /** Service that hangs to trigger an ANR. */ 628 private class AnrGenerator extends Service { 629 @Override 630 public IBinder onBind(Intent intent) { 631 return null; 632 } 633 634 @Override 635 public int onStartCommand(Intent intent, int flags, int id) { 636 Log.i(LOG_TAG, "in service start -- about to hang"); 637 try { 638 Thread.sleep(ANR_DELAY); 639 } catch (InterruptedException e) { 640 Log.wtf(LOG_TAG, e); 641 } 642 Log.i(LOG_TAG, "service hang finished -- stopping and returning"); 643 stopSelf(); 644 return START_NOT_STICKY; 645 } 646 } 647 } 648 649 /** 650 * Collect a screenshot on test failure. 651 */ 652 private class Screenshotter extends AuptListener { 653 private void collectScreenshot(Test test, String suffix) { 654 UiDevice device = UiDevice.getInstance(AuptTestRunner.this); 655 656 if (device == null) { 657 Log.w(LOG_TAG, "Couldn't collect screenshot on test failure"); 658 return; 659 } 660 661 String testName = 662 test instanceof TestCase 663 ? ((TestCase) test).getName() 664 : (test instanceof TestSuite ? ((TestSuite) test).getName() : test.toString()); 665 666 String fileName = 667 mResultsDirectory.getPath() 668 + "/" + testName.replaceAll(".", "_") 669 + suffix + ".png"; 670 671 device.takeScreenshot(new File(fileName)); 672 } 673 674 @Override 675 public void addError(Test test, Throwable t) { 676 collectScreenshot(test, 677 "_failure_screenshot_" + SCREENSHOT_DATE_FORMAT.format(new Date())); 678 } 679 680 @Override 681 public void addFailure(Test test, AssertionFailedError t) { 682 collectScreenshot(test, 683 "_failure_screenshot_" + SCREENSHOT_DATE_FORMAT.format(new Date())); 684 } 685 } 686 687 /** Runs a command when a test finishes. */ 688 private class FinishCommandRunner extends AuptListener { 689 @Override 690 public void endTest(Test test) { 691 for (String command : mFinishCommands) { 692 AuptTestRunner.this.getUiAutomation().executeShellCommand(command); 693 } 694 } 695 } 696 } 697