1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.uiautomator.platform; 18 19 import android.os.Bundle; 20 import android.os.Environment; 21 import android.util.Log; 22 23 import com.android.uiautomator.core.UiDevice; 24 import com.android.uiautomator.testrunner.UiAutomatorTestCase; 25 26 import java.io.BufferedOutputStream; 27 import java.io.BufferedReader; 28 import java.io.BufferedWriter; 29 import java.io.File; 30 import java.io.FileInputStream; 31 import java.io.FileNotFoundException; 32 import java.io.FileOutputStream; 33 import java.io.FileWriter; 34 import java.io.IOException; 35 import java.io.InputStream; 36 import java.io.InputStreamReader; 37 import java.util.ArrayList; 38 import java.util.List; 39 import java.util.Properties; 40 41 /** 42 * Base class for jank test. 43 * All jank test needs to extend JankTestBase 44 */ 45 public class JankTestBase extends UiAutomatorTestCase { 46 private static final String TAG = JankTestBase.class.getSimpleName(); 47 48 protected UiDevice mDevice; 49 protected TestWatchers mTestWatchers = null; 50 protected BufferedWriter mWriter = null; 51 protected BufferedWriter mStatusWriter = null; 52 protected int mIteration = 20; // default iteration is set 20 53 /* can be used to enable/disable systrace in the test */ 54 protected int mTraceTime = 0; 55 protected Bundle mParams; 56 protected String mTestCaseName; 57 protected int mSuccessTestRuns = 0; 58 protected Thread mThread = null; 59 60 // holds all params for the derived tests 61 private static final String PROPERTY_FILE_NAME = "UiJankinessTests.conf"; 62 private static final String PARAM_CONFIG = "conf"; 63 private static final String LOCAL_TMP_DIR = "/data/local/tmp/"; 64 // File that hold the test results 65 private static String OUTPUT_FILE_NAME = LOCAL_TMP_DIR + "UiJankinessTestsOutput.txt"; 66 // File that hold test status, e.g successful test iterations 67 private static String STATUS_FILE_NAME = LOCAL_TMP_DIR + "UiJankinessTestsStatus.txt"; 68 private static final String RAW_DATA_DIR = LOCAL_TMP_DIR + "UiJankinessRawData"; 69 70 private static int SUCCESS_THRESHOLD = 80; 71 private static boolean DEBUG = false; 72 73 /* default animation time is set to 2 seconds */ 74 protected static final long DEFAULT_ANIMATION_TIME = 2 * 1000; 75 /* default swipe steps for fling animation */ 76 protected static final int DEFAULT_FLING_STEPS = 8; 77 78 /* Array to record jankiness data in each test iteration */ 79 private int[] jankinessArray; 80 /* Array to record frame rate in each test iteration */ 81 private double[] frameRateArray; 82 /* Array to save max accumulated frame number in each test iteration */ 83 private int[] maxDeltaVsyncArray; 84 /* Default file to store the systrace */ 85 private static final File SYSTRACE_DIR = new File(LOCAL_TMP_DIR, "systrace"); 86 /* Default trace file name */ 87 private static final String TRACE_FILE_NAME = "trace.txt"; 88 /* Default tracing time is 5 seconds */ 89 private static final int DEFAULT_TRACE_TIME = 5; // 5 seconds 90 // Command to dump compressed trace data 91 private static final String ATRACE_COMMAND = "atrace -z -t %d gfx input view sched freq"; 92 93 /** 94 * Thread to capture systrace log from the test 95 */ 96 public class SystraceTracker implements Runnable { 97 File mFile = new File(SYSTRACE_DIR, TRACE_FILE_NAME); 98 int mTime = DEFAULT_TRACE_TIME; 99 100 public SystraceTracker(int traceTime, String fileName) { 101 try { 102 if (!SYSTRACE_DIR.exists()) { 103 if (!SYSTRACE_DIR.mkdir()) { 104 log(String.format("create directory %s failed, you can manually create " 105 + "it and start the test again", SYSTRACE_DIR.getAbsolutePath())); 106 return; 107 } 108 } 109 } catch (SecurityException e) { 110 Log.e(TAG, "creating directory failed?", e); 111 } 112 113 if (traceTime > 0) { 114 mTime = traceTime; 115 } 116 if (fileName != null) { 117 mFile = new File(SYSTRACE_DIR, fileName); 118 } 119 } 120 121 @Override 122 public void run() { 123 String command = String.format(ATRACE_COMMAND, mTime); 124 Log.v(TAG, "command: " + command); 125 Process p = null; 126 InputStream in = null; 127 BufferedOutputStream out = null; 128 try { 129 p = Runtime.getRuntime().exec(command); 130 Log.v(TAG, "write systrace into file: " + mFile.getAbsolutePath()); 131 // read bytes from the process output stream as the output is compressed 132 byte[] buffer = new byte[1024]; 133 in = p.getInputStream(); 134 out = new BufferedOutputStream(new FileOutputStream(mFile)); 135 int n; 136 while ((n = in.read(buffer)) != -1) { 137 out.write(buffer, 0, n); 138 out.flush(); 139 } 140 in.close(); 141 out.close(); 142 // read error message 143 BufferedReader br = new BufferedReader(new InputStreamReader(p.getErrorStream())); 144 String line; 145 while ((line = br.readLine()) != null) { 146 Log.e(TAG, "Command return errors: " + line); 147 } 148 br.close(); 149 150 // Due to limited buffer size for standard input and output stream, 151 // promptly reading from the input stream or output stream to avoid block 152 int status = p.waitFor(); 153 if (status != 0) { 154 Log.e(TAG, String.format("Run shell command: %s, status: %s", 155 command, status)); 156 } 157 } catch (InterruptedException e) { 158 Log.e(TAG, "Exception from command " + command + ":"); 159 Log.e(TAG, "Thread interrupted? ", e); 160 } catch (IOException e) { 161 Log.e(TAG, "Open file error: ", e); 162 } catch (IllegalThreadStateException e) { 163 Log.e(TAG, "the process has not exit yet ", e); 164 } 165 } 166 } 167 168 @Override 169 protected void setUp() throws Exception { 170 super.setUp(); 171 mDevice = UiDevice.getInstance(); 172 mTestWatchers = new TestWatchers(); // extends the common class UiWatchers 173 mTestWatchers.registerAnrAndCrashWatchers(); 174 175 mWriter = new BufferedWriter(new FileWriter(new File(OUTPUT_FILE_NAME), true)); 176 mStatusWriter = new BufferedWriter(new FileWriter(new File(STATUS_FILE_NAME), true)); 177 178 mParams = getParams(); 179 if (mParams != null && !mParams.isEmpty()) { 180 log("mParams is not empty, get properties."); 181 String mIterationStr = getPropertyString(mParams, "iteration"); 182 if (mIterationStr != null) { 183 mIteration = Integer.valueOf(mIterationStr); 184 } 185 String mTraceTimeStr = getPropertyString(mParams, "tracetime"); 186 if (mTraceTimeStr != null) { 187 mTraceTime = Integer.valueOf(mTraceTimeStr); 188 } 189 } 190 jankinessArray = new int[mIteration]; 191 frameRateArray = new double[mIteration]; 192 maxDeltaVsyncArray = new int[mIteration]; 193 mTestCaseName = this.getName(); 194 195 mSuccessTestRuns = 0; 196 mDevice.pressHome(); 197 } 198 199 /** 200 * Create a new thread for systrace and start the thread 201 * 202 * @param testCaseName 203 * @param iteration 204 */ 205 protected void startTrace(String testCaseName, int iteration) { 206 if (mTraceTime > 0) { 207 String outputFile = String.format("%s_%d_trace", mTestCaseName, iteration); 208 mThread = new Thread(new SystraceTracker(mTraceTime, outputFile)); 209 mThread.start(); 210 } 211 } 212 213 /** 214 * Wait for the tracing thread to exit 215 */ 216 protected void endTrace() { 217 if (mThread != null) { 218 try { 219 mThread.join(); 220 } catch (InterruptedException e) { 221 Log.e(TAG, "wait for the trace thread to exit exception:", e); 222 } 223 } 224 } 225 226 /** 227 * Expects a file from the command line via conf param or default following format each on its 228 * own line. <code> 229 * key=Value 230 * Browser_URL1=cnn.com 231 * Browser_URL2=google.com 232 * Camera_ShutterDelay=1000 233 * etc... 234 * </code> 235 * @param Bundle params 236 * @param key 237 * @return the value of the property else defaultValue 238 * @throws FileNotFoundException 239 * @throws IOException 240 */ 241 protected String getPropertyString(Bundle params, String key) 242 throws FileNotFoundException, IOException { 243 Properties prop = new Properties(); 244 prop.load(new FileInputStream(new File(LOCAL_TMP_DIR, 245 params.getString(PARAM_CONFIG, PROPERTY_FILE_NAME)))); 246 String value = prop.getProperty(key); 247 if (value != null && !value.isEmpty()) 248 return value; 249 return null; 250 } 251 252 /** 253 * Expects a file from the command line via conf param or default following format each on its 254 * own line. <code> 255 * key=Value 256 * Browser_URL1=cnn.com 257 * Browser_URL2=google.com 258 * Camera_ShutterDelay=1000 259 * etc... 260 * </code> 261 * @param Bundle params 262 * @param key 263 * @return the value of the property else defaultValue 264 * @throws FileNotFoundException 265 * @throws IOException 266 */ 267 protected long getPropertyLong(Bundle params, String key) 268 throws FileNotFoundException, IOException { 269 Properties prop = new Properties(); 270 prop.load(new FileInputStream(new File(LOCAL_TMP_DIR, 271 params.getString(PARAM_CONFIG, PROPERTY_FILE_NAME)))); 272 String value = prop.getProperty(key); 273 if (value != null && !value.trim().isEmpty()) 274 return Long.valueOf(value.trim()); 275 return 0; 276 } 277 278 /** 279 * Verify the test result by comparing data sample size with expected value 280 * @param expectedDataSize the expected data size 281 */ 282 protected boolean validateResults(int expectedDataSize) { 283 int receivedDataSize = SurfaceFlingerHelper.getDataSampleSize(); 284 return ((expectedDataSize > 0) && (receivedDataSize >= expectedDataSize)); 285 } 286 287 /** 288 * Process the raw data, calculate jankiness, frame rate and max accumulated frames number 289 * @param testCaseName 290 * @param iteration 291 */ 292 protected void recordResults(String testCaseName, int iteration) { 293 long refreshPeriod = SurfaceFlingerHelper.getRefreshPeriod(); 294 // if the raw directory doesn't exit, create the directory 295 File rawDataDir = new File(RAW_DATA_DIR); 296 try { 297 if (!rawDataDir.exists()) { 298 if (!rawDataDir.mkdir()) { 299 log(String.format("create directory %s failed, you can manually create " + 300 "it and start the test again", rawDataDir)); 301 } 302 } 303 } catch (SecurityException e) { 304 Log.e(TAG, "create directory failed: ", e); 305 } 306 String rawFileName = String.format("%s/%s_%d.txt", RAW_DATA_DIR, testCaseName, iteration); 307 // write results into a file 308 BufferedWriter fw = null; 309 try { 310 fw = new BufferedWriter(new FileWriter(new File(rawFileName), false)); 311 fw.write(SurfaceFlingerHelper.getFrameBufferData()); 312 } catch (IOException e) { 313 Log.e(TAG, "failed to write to file", e); 314 return; 315 } finally { 316 try { 317 if (fw != null) { 318 fw.close(); 319 } 320 } 321 catch (IOException e) { 322 Log.e(TAG, "close file failed.", e); 323 } 324 } 325 326 // get jankiness count 327 int jankinessCount = SurfaceFlingerHelper.getVsyncJankiness(); 328 // get frame rate 329 double frameRate = SurfaceFlingerHelper.getFrameRate(); 330 // get max accumulated frames 331 int maxDeltaVsync = SurfaceFlingerHelper.getMaxDeltaVsync(); 332 333 // only record data when they are valid 334 if (jankinessCount >=0 && frameRate > 0) { 335 jankinessArray[iteration] = jankinessCount; 336 frameRateArray[iteration] = frameRate; 337 maxDeltaVsyncArray[iteration] = maxDeltaVsync; 338 mSuccessTestRuns++; 339 } 340 String msg = String.format("%s, iteration %d\n" + 341 "refresh period: %d\n" + 342 "jankiness count: %d\n" + 343 "frame rate: %f\n" + 344 "max accumulated frames: %d\n", 345 testCaseName, iteration, refreshPeriod, 346 jankinessCount, frameRate, maxDeltaVsync); 347 log(msg); 348 if (DEBUG) { 349 SurfaceFlingerHelper.printData(testCaseName, iteration); 350 } 351 } 352 353 /** 354 * Process data from all test iterations, and save to disk 355 * @param testCaseName 356 */ 357 protected void saveResults(String testCaseName) { 358 // write test status into status file 359 try { 360 mStatusWriter.write(String.format("%s: %d success runs out of %d iterations\n", 361 testCaseName, mSuccessTestRuns, mIteration)); 362 } catch (IOException e) { 363 log("failed to write output for test case " + testCaseName); 364 } 365 366 // if successful test runs is less than the threshold, no results will be saved. 367 if (mSuccessTestRuns * 100 / mIteration < SUCCESS_THRESHOLD) { 368 log(String.format("In %s, # of successful test runs out of %s iterations: %d ", 369 testCaseName, mIteration, mSuccessTestRuns)); 370 log(String.format("threshold is %d%%", SUCCESS_THRESHOLD)); 371 return; 372 } 373 374 if (DEBUG) { 375 print(jankinessArray, "jankiness array"); 376 print(frameRateArray, "frame rate array"); 377 print(maxDeltaVsyncArray, "max delta vsync array"); 378 } 379 double avgJankinessCount = getAverage(jankinessArray); 380 int maxJankinessCount = getMaxValue(jankinessArray); 381 double avgFrameRate = getAverage(frameRateArray); 382 double avgMaxDeltaVsync = getAverage(maxDeltaVsyncArray); 383 384 String avgMsg = String.format("%s\n" + 385 "average number of jankiness: %f\n" + 386 "max number of jankiness: %d\n" + 387 "average frame rate: %f\n" + 388 "average of max accumulated frames: %f\n", 389 testCaseName, avgJankinessCount, maxJankinessCount, avgFrameRate, avgMaxDeltaVsync); 390 log(avgMsg); 391 392 try { 393 mWriter.write(avgMsg); 394 } catch (IOException e) { 395 log("failed to write output for test case " + testCaseName); 396 } 397 } 398 399 // return the max value in an integer array 400 private int getMaxValue(int[] intArray) { 401 int index = 0; 402 int max = intArray[index]; 403 for (int i = 1; i < intArray.length; i++) { 404 if (max < intArray[i]) { 405 max = intArray[i]; 406 } 407 } 408 return max; 409 } 410 411 private double getAverage(int[] intArray) { 412 int mean = 0; 413 int numberTests = 0; 414 for (int i = 0; i < intArray.length; i++) { 415 // in case in some iteration, test fails, no data points is collected 416 if (intArray[i] >= 0) { 417 mean += intArray[i]; 418 ++numberTests; 419 } 420 } 421 return (double)mean/numberTests; 422 } 423 424 private double getAverage(double[] doubleArray) { 425 double mean = 0; 426 int numberTests = 0; 427 for (int i = 0; i < doubleArray.length; i++) { 428 // in case in some iteration, test fails, no data points is collected 429 if (doubleArray[i] >= 0) { 430 mean += doubleArray[i]; 431 ++numberTests; 432 } 433 } 434 return mean/numberTests; 435 } 436 437 private void print(int[] intArray, String arrayName) { 438 log("start to print array for " + arrayName); 439 for (int i = 0; i < intArray.length; i++) { 440 log(String.format("%d: %d", i, intArray[i])); 441 } 442 } 443 444 private void print(double[] doubleArray, String arrayName) { 445 log("start to print array for " + arrayName); 446 for (int i = 0; i < doubleArray.length; i++) { 447 log(String.format("%d: %f", i, doubleArray[i])); 448 } 449 } 450 451 @Override 452 protected void tearDown() throws Exception { 453 super.tearDown(); 454 if (mWriter != null) { 455 mWriter.close(); 456 } 457 if (mStatusWriter != null) { 458 mStatusWriter.close(); 459 } 460 } 461 462 private void log(String message) { 463 Log.v(TAG, message); 464 } 465 466 /** 467 * Set the total number of test iteration 468 * @param iteration 469 */ 470 protected void setIteration(int iteration){ 471 mIteration = iteration; 472 } 473 474 /** 475 * Get the total number of test iteration 476 * @return iteration 477 */ 478 protected int getIteration(){ 479 return mIteration; 480 } 481 } 482