1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.graphics.tests; 18 19 import com.android.tradefed.config.Option; 20 import com.android.tradefed.config.Option.Importance; 21 import com.android.tradefed.device.CollectingOutputReceiver; 22 import com.android.tradefed.device.DeviceNotAvailableException; 23 import com.android.tradefed.device.ITestDevice; 24 import com.android.tradefed.log.LogUtil.CLog; 25 import com.android.tradefed.result.ITestInvocationListener; 26 import com.android.tradefed.testtype.IDeviceTest; 27 import com.android.tradefed.testtype.IRemoteTest; 28 import com.android.tradefed.util.AbiFormatter; 29 import com.android.tradefed.util.RunUtil; 30 import com.android.tradefed.util.proto.TfMetricProtoUtil; 31 32 import org.junit.Assert; 33 34 import java.util.HashMap; 35 import java.util.Map; 36 import java.util.concurrent.TimeUnit; 37 38 /** 39 * Test Runner for graphics Flatland Benchmark test. 40 * <p> 41 * Flatland test is a benchmark for measuring GPU performance in various 2D UI rendering and 42 * window composition scenarios. 43 * <p> 44 * Since it is measuring the hardware performance, the test should be executed in 45 * as consistent and static an environment as possible. 46 * <ul> 47 * <li>The display should be turned off and background service should be stopped before 48 * running the benchmark. Running 'adb shell stop' is probably sufficient for this, 49 * but if there are device specific background services that consume 50 * much CPU cycles, memory bandwidth, or might otherwise interfere with GPU rendering, 51 * those should be stopped as well 52 * <li>All relevant hardware clocks should be locked at particular frequency when running the test. 53 * </ul> 54 * <p> 55 * If running the benchmark with clocks locked causes thermal throttling, set option "--sleep-time" 56 * to 10 to 50 (ms) to insert sleep between each benchmark sample run. 57 * <p> 58 * Output interpretation: 59 * For each test case, the expected time in milliseconds that a single frame of the scenario 60 * takes to complete will be printed out. Four types of values could displayed: 61 * <ul> 62 * <li>fast - frames of the scenarios are completed too fast to be reliably benchmarked. This 63 * corresponds to frame time less than 3 ms. The scenario was skipped. "0" will be posted into 64 * the dashboard. 65 * <li>slow - frame time is too long, normally orver 50 ms. The scenario was skipped. "1000" will 66 * be posted into the dashboard. 67 * <li>varies - frame time was not stable. rerun the test to get a stable results. If that results 68 * show repeatedly, something is wrong with the environment, signal to file a bug. 69 * <li>decimal number - frame time for the scenarios are measured. 70 * </ul> 71 */ 72 public class FlatlandTest implements IDeviceTest, IRemoteTest { 73 74 private static final long SHELL_TIMEOUT = 30*60*1000; 75 private static final String COMMAND = "flatland|#ABI32#|"; 76 private static final String FIRST_LINE = "cmdline:"; 77 private static final String TITLE = "Scenario"; 78 private static final long START_TIMER = 2 * 60 * 1000; // 2 minutes 79 private static final String RESULT_FAST = "fast"; 80 private static final String RESULT_SLOW = "slow"; 81 private static final String RESULT_VARIES = "varies"; 82 83 private ITestDevice mTestDevice = null; 84 // HashMap to store results for 85 public Map<String, String> mResultMap = new HashMap<String, String>(); 86 87 @Option(name = "ru-key", description = "Reporting unit key to use when posting results") 88 private String mRuKey = "flatland"; 89 90 @Option(name = "run-path", 91 description = "path for the binary") 92 private String mRunPath = "/data/local/tmp/"; 93 94 @Option(name = "sleep-time", 95 description = "sleep for N ms between samples, set to 10 - 50 ms if the locked CPU" 96 + " frequency causes thermal throttle.") 97 private int mSleepTime = 50; 98 99 @Option(name = "schema-map", 100 description = "map a test case name to a schema key") 101 private Map<String, String> mSchemaMap = new HashMap<String, String>(); 102 103 @Option(name = AbiFormatter.FORCE_ABI_STRING, 104 description = AbiFormatter.FORCE_ABI_DESCRIPTION, 105 importance = Importance.IF_UNSET) 106 private String mForceAbi = null; 107 108 @Override 109 public void setDevice(ITestDevice testDevice) { 110 mTestDevice = testDevice; 111 } 112 113 @Override 114 public ITestDevice getDevice() { 115 return mTestDevice; 116 } 117 118 @Override 119 public void run(ITestInvocationListener standardListener) throws DeviceNotAvailableException { 120 Assert.assertNotNull(mRunPath); 121 RunUtil.getDefault().sleep(START_TIMER); 122 123 // execute test 124 StringBuilder cmd = new StringBuilder(); 125 cmd.append(mRunPath); 126 cmd.append(COMMAND); 127 if (mSleepTime > 0) { 128 cmd.append(" -s "); 129 cmd.append(mSleepTime); 130 } 131 standardListener.testRunStarted(mRuKey, 1); 132 long start = System.currentTimeMillis(); 133 CollectingOutputReceiver receiver = new CollectingOutputReceiver(); 134 mTestDevice.executeShellCommand(AbiFormatter.formatCmdForAbi(cmd.toString(), mForceAbi), 135 receiver, SHELL_TIMEOUT, TimeUnit.MILLISECONDS, 2); 136 String result = receiver.getOutput(); 137 if (result == null) { 138 CLog.v("no test results returned. Test failed?"); 139 return; 140 } 141 // parse results and report metrics 142 parseResult(result); 143 standardListener.testRunEnded( 144 (System.currentTimeMillis() - start), TfMetricProtoUtil.upgradeConvert(mResultMap)); 145 } 146 147 /** 148 * Parse results returned from running the benchmark 149 */ 150 public void parseResult(String result) { 151 String[] lines = result.split(System.getProperty("line.separator")); 152 if (lines.length <= 0) { 153 return; 154 } 155 for (int i = 0; i < lines.length; i++) { 156 if (!lines[i].contains(FIRST_LINE) && !(lines[i].contains(TITLE))) { 157 // skip the first two lines 158 String[] items = lines[i].trim().split("\\|"); 159 if (items.length == 3) { 160 String schemaKey = String.format("%s %s", items[0].trim(), items[1].trim()); 161 if (mSchemaMap.get(schemaKey) != null) { 162 // get the mapped schema key if there is any 163 schemaKey = mSchemaMap.get(schemaKey); 164 } 165 String renderTime = items[2].trim(); 166 if (renderTime != null) { 167 if (renderTime.equals(RESULT_FAST)) { 168 mResultMap.put(schemaKey, "0"); 169 } else if (renderTime.equals(RESULT_SLOW)) { 170 mResultMap.put(schemaKey, "1000"); 171 } else if (renderTime.equals(RESULT_VARIES)){ 172 mResultMap.put(schemaKey, "-1"); 173 } else { 174 mResultMap.put(schemaKey, renderTime); 175 } 176 } 177 } 178 } 179 } 180 } 181 } 182
