1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.graphics.tests; 18 19 import com.android.ddmlib.IDevice; 20 import com.android.ddmlib.Log; 21 import com.android.ddmlib.NullOutputReceiver; 22 import com.android.tradefed.config.Option; 23 import com.android.tradefed.device.DeviceNotAvailableException; 24 import com.android.tradefed.device.ITestDevice; 25 import com.android.tradefed.result.FileInputStreamSource; 26 import com.android.tradefed.result.ITestInvocationListener; 27 import com.android.tradefed.result.InputStreamSource; 28 import com.android.tradefed.result.LogDataType; 29 import com.android.tradefed.testtype.IDeviceTest; 30 import com.android.tradefed.testtype.IRemoteTest; 31 import com.android.tradefed.util.FileUtil; 32 import com.android.tradefed.util.RunUtil; 33 import com.android.tradefed.util.StreamUtil; 34 35 import org.junit.Assert; 36 37 import java.io.File; 38 import java.io.FileInputStream; 39 import java.io.IOException; 40 import java.io.InputStream; 41 import java.util.ArrayList; 42 import java.util.Arrays; 43 import java.util.Collection; 44 import java.util.HashMap; 45 import java.util.List; 46 import java.util.Map; 47 import java.util.concurrent.TimeUnit; 48 import java.util.regex.Matcher; 49 import java.util.regex.Pattern; 50 51 /** 52 * Runs the user action framerate benchmark test. This test capture 53 * use the scripted monkey to inject the keyevent to mimic the real 54 * user action, capture the average framerate and save the result 55 * file to the sdcard. 56 * <p/> 57 * Note that this test will not run properly unless /sdcard is mounted and 58 * writable. 59 */ 60 public class UserActionBenchmark implements IDeviceTest, IRemoteTest { 61 private static final String LOG_TAG = "UserActionBenchmark"; 62 63 ITestDevice mTestDevice = null; 64 65 private static final long START_TIMER = 2 * 60 * 1000; // 2 minutes 66 67 @Option(name = "test-output-filename", description = "The test output filename.") 68 private String mDeviceTestOutputFilename = "avgFrameRateOut.txt"; 69 70 // The time in ms to wait the scripted monkey finish. 71 private static final int CMD_TIMEOUT = 60 * 60 * 1000; 72 73 private static final Pattern AVERAGE_FPS = Pattern.compile("(.*):(\\d+.\\d+)"); 74 75 @Option(name = "test-case", description = "The name of test-cases to run. May be repeated.") 76 private Collection<String> mTestCases = new ArrayList<>(); 77 78 @Option(name = "iteration", description = "Test run iteration") 79 private int mIteration = 1; 80 81 @Option(name = "throttle", description = "Scripted monkey throttle time") 82 private int mThrottle = 500; // in milliseconds 83 84 @Option(name = "script-path", description = "Test script path") 85 private String mScriptPath = "userActionFPSScript"; 86 87 @Option(name = "test-label", description = "Test label") 88 private String mTestLabel = "UserActionFramerateBenchmark"; 89 90 @Override 91 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 92 Assert.assertNotNull(mTestDevice); 93 94 // Start the test after device is fully booted and stable 95 // FIXME: add option in TF to wait until device is booted and stable 96 RunUtil.getDefault().sleep(START_TIMER); 97 98 String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); 99 String scriptFullPath = 100 String.format("%s/%s/%s", extStore, mScriptPath, mTestDevice.getProductType()); 101 for (String testCase : mTestCases) { 102 // Start the scripted monkey command 103 mTestDevice.executeShellCommand(String.format( 104 "monkey -f /%s/%s.txt --throttle %d %d", scriptFullPath, 105 testCase, mThrottle, mIteration), new NullOutputReceiver(), 106 CMD_TIMEOUT, TimeUnit.MILLISECONDS, 2); 107 logOutputFiles(listener); 108 cleanResultFile(); 109 } 110 } 111 112 /** 113 * Clean up the test result file from test run 114 */ 115 private void cleanResultFile() throws DeviceNotAvailableException { 116 String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); 117 mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, 118 mDeviceTestOutputFilename)); 119 } 120 121 /** 122 * Pull the output files from the device, add it to the logs, and also parse 123 * out the relevant test metrics and report them. 124 */ 125 private void logOutputFiles(ITestInvocationListener listener) 126 throws DeviceNotAvailableException { 127 File outputFile = null; 128 InputStreamSource outputSource = null; 129 try { 130 outputFile = mTestDevice.pullFileFromExternal(mDeviceTestOutputFilename); 131 132 if (outputFile == null) { 133 return; 134 } 135 136 // Upload a verbatim copy of the output file 137 Log.d(LOG_TAG, String.format("Sending %d byte file %s into the logosphere!", 138 outputFile.length(), outputFile)); 139 outputSource = new FileInputStreamSource(outputFile); 140 listener.testLog(mDeviceTestOutputFilename, LogDataType.TEXT, outputSource); 141 142 // Parse the output file to upload aggregated metrics 143 parseOutputFile(new FileInputStream(outputFile), listener); 144 } catch (IOException e) { 145 Log.e(LOG_TAG, String.format( 146 "IOException while reading or parsing output file: %s", e)); 147 } finally { 148 FileUtil.deleteFile(outputFile); 149 StreamUtil.cancel(outputSource); 150 } 151 } 152 153 /** 154 * Parse the test result, calculate the average and parse the metrics 155 * from the scripted monkey test output file 156 */ 157 private void parseOutputFile(InputStream dataStream, 158 ITestInvocationListener listener) { 159 160 Map<String, String> runMetrics = new HashMap<>(); 161 162 // try to parse it 163 String contents; 164 try { 165 contents = StreamUtil.getStringFromStream(dataStream); 166 } catch (IOException e) { 167 Log.e(LOG_TAG, String.format( 168 "Got IOException during test processing: %s", e)); 169 return; 170 } 171 172 List<String> lines = Arrays.asList(contents.split("\n")); 173 String key = null; 174 float averageResult; 175 float totalResult = 0; 176 int counter = 0; 177 178 // collect the result and calculate the average 179 for (String line: lines) { 180 Matcher m = AVERAGE_FPS.matcher(line); 181 if (m.matches()) { 182 key = m.group(1); 183 totalResult += Float.parseFloat(m.group(2)); 184 counter++; 185 } 186 } 187 averageResult = totalResult / counter; 188 Log.i(LOG_TAG, String.format("averageResult = %s\n", averageResult)); 189 runMetrics.put(key, Float.toString(averageResult)); 190 reportMetrics(listener, runMetrics); 191 } 192 193 /** 194 * Report run metrics by creating an empty test run to stick them in 195 * <p /> 196 * Exposed for unit testing 197 */ 198 void reportMetrics(ITestInvocationListener listener, Map<String, String> metrics) { 199 Log.d(LOG_TAG, String.format("About to report metrics: %s", metrics)); 200 listener.testRunStarted(mTestLabel, 0); 201 listener.testRunEnded(0, metrics); 202 } 203 204 @Override 205 public void setDevice(ITestDevice device) { 206 mTestDevice = device; 207 } 208 209 @Override 210 public ITestDevice getDevice() { 211 return mTestDevice; 212 } 213 } 214