1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.performance.tests; 18 19 import com.android.tradefed.config.Option; 20 import com.android.tradefed.config.Option.Importance; 21 import com.android.tradefed.device.DeviceNotAvailableException; 22 import com.android.tradefed.device.ITestDevice; 23 import com.android.tradefed.log.LogUtil.CLog; 24 import com.android.tradefed.result.ITestInvocationListener; 25 import com.android.tradefed.result.TestDescription; 26 import com.android.tradefed.testtype.IDeviceTest; 27 import com.android.tradefed.testtype.IRemoteTest; 28 import com.android.tradefed.util.RunUtil; 29 import com.android.tradefed.util.proto.TfMetricProtoUtil; 30 31 import org.w3c.dom.Document; 32 import org.w3c.dom.Element; 33 import org.w3c.dom.Node; 34 import org.w3c.dom.NodeList; 35 import org.xml.sax.SAXException; 36 37 import java.io.File; 38 import java.io.IOException; 39 import java.util.Collections; 40 import java.util.HashMap; 41 import java.util.Map; 42 43 import javax.xml.parsers.DocumentBuilder; 44 import javax.xml.parsers.DocumentBuilderFactory; 45 import javax.xml.parsers.ParserConfigurationException; 46 47 /** 48 * A harness that launches GLBenchmark and reports result. Requires GLBenchmark 49 * custom XML. Assumes GLBenchmark app is already installed on device. 50 */ 51 public class GLBenchmarkTest implements IDeviceTest, IRemoteTest { 52 53 private static final String RUN_KEY = "glbenchmark"; 54 private static final long TIMEOUT_MS = 30 * 60 * 1000; 55 private static final long POLLING_INTERVAL_MS = 5 * 1000; 56 private static final Map<String, String> METRICS_KEY_MAP = createMetricsKeyMap(); 57 58 private ITestDevice mDevice; 59 60 @Option(name = "custom-xml-path", description = "local path for GLBenchmark custom.xml", 61 importance = Importance.ALWAYS) 62 private File mGlbenchmarkCustomXmlLocal = new File("/tmp/glbenchmark_custom.xml"); 63 64 @Option(name = "gl-package-name", description = "GLBenchmark package name") 65 private String mGlbenchmarkPackageName = "com.glbenchmark.glbenchmark25"; 66 67 @Option(name = "gl-version", description = "GLBenchmark version (e.g. 2.5.1_b306a5)") 68 private String mGlbenchmarkVersion = "2.5.1_b306a5"; 69 70 private String mGlbenchmarkCacheDir = 71 "${EXTERNAL_STORAGE}/Android/data/" + mGlbenchmarkPackageName + "/cache/"; 72 private String mGlbenchmarkCustomXmlPath = 73 mGlbenchmarkCacheDir + "custom.xml"; 74 private String mGlbenchmarkResultXmlPath = 75 mGlbenchmarkCacheDir + "last_results_" + mGlbenchmarkVersion + ".xml"; 76 private String mGlbenchmarkExcelResultXmlPath = 77 mGlbenchmarkCacheDir + "results_%s_0.xml"; 78 private String mGlbenchmarkAllResultXmlPath = 79 mGlbenchmarkCacheDir + "results*.xml"; 80 81 private static Map<String, String> createMetricsKeyMap() { 82 Map<String, String> result = new HashMap<String, String>(); 83 result.put("Fill rate - C24Z16", "fill-rate"); 84 result.put("Fill rate - C24Z16 Offscreen", "fill-rate-offscreen"); 85 result.put("Triangle throughput: Textured - C24Z16", "triangle-c24z16"); 86 result.put("Triangle throughput: Textured - C24Z16 Offscreen", 87 "triangle-c24z16-offscreen"); 88 result.put("Triangle throughput: Textured - C24Z16 Vertex lit", 89 "triangle-c24z16-vertex-lit"); 90 result.put("Triangle throughput: Textured - C24Z16 Offscreen Vertex lit", 91 "triangle-c24z16-offscreen-vertex-lit"); 92 result.put("Triangle throughput: Textured - C24Z16 Fragment lit", 93 "triangle-c24z16-fragment-lit"); 94 result.put("Triangle throughput: Textured - C24Z16 Offscreen Fragment lit", 95 "triangle-c24z16-offscreen-fragment-lit"); 96 result.put("GLBenchmark 2.5 Egypt HD - C24Z16", "egypt-hd-c24z16"); 97 result.put("GLBenchmark 2.5 Egypt HD - C24Z16 Offscreen", "egypt-hd-c24z16-offscreen"); 98 result.put("GLBenchmark 2.5 Egypt HD PVRTC4 - C24Z16", "egypt-hd-pvrtc4-c24z16"); 99 result.put("GLBenchmark 2.5 Egypt HD PVRTC4 - C24Z16 Offscreen", 100 "egypt-hd-pvrtc4-c24z16-offscreen"); 101 result.put("GLBenchmark 2.5 Egypt HD - C24Z24MS4", "egypt-hd-c24z24ms4"); 102 result.put("GLBenchmark 2.5 Egypt HD - C24Z16 Fixed timestep", 103 "egypt-hd-c24z16-fixed-timestep"); 104 result.put("GLBenchmark 2.5 Egypt HD - C24Z16 Fixed timestep Offscreen", 105 "egypt-hd-c24z16-fixed-timestep-offscreen"); 106 result.put("GLBenchmark 2.1 Egypt Classic - C16Z16", "egypt-classic-c16z16"); 107 result.put("GLBenchmark 2.1 Egypt Classic - C16Z16 Offscreen", 108 "egypt-classic-c16z16-offscreen"); 109 return Collections.unmodifiableMap(result); 110 } 111 112 /** 113 * {@inheritDoc} 114 */ 115 @Override 116 public void setDevice(ITestDevice device) { 117 mDevice = device; 118 } 119 120 /** 121 * {@inheritDoc} 122 */ 123 @Override 124 public ITestDevice getDevice() { 125 return mDevice; 126 } 127 128 /** 129 * {@inheritDoc} 130 */ 131 @Override 132 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 133 TestDescription testId = new TestDescription(getClass().getCanonicalName(), RUN_KEY); 134 ITestDevice device = getDevice(); 135 136 // delete old result 137 device.executeShellCommand(String.format("rm %s", mGlbenchmarkResultXmlPath)); 138 device.executeShellCommand(String.format("rm %s", mGlbenchmarkAllResultXmlPath)); 139 140 // push glbenchmark custom xml to device 141 device.pushFile(mGlbenchmarkCustomXmlLocal, mGlbenchmarkCustomXmlPath); 142 143 listener.testRunStarted(RUN_KEY, 0); 144 listener.testStarted(testId); 145 146 long testStartTime = System.currentTimeMillis(); 147 boolean isRunningBenchmark; 148 149 boolean isTimedOut = false; 150 boolean isResultGenerated = false; 151 Map<String, String> metrics = new HashMap<String, String>(); 152 String errMsg = null; 153 154 String deviceModel = device.executeShellCommand("getprop ro.product.model"); 155 String resultExcelXmlPath = String.format(mGlbenchmarkExcelResultXmlPath, 156 deviceModel.trim().replaceAll("[ -]", "_").toLowerCase()); 157 CLog.i("Result excel xml path:" + resultExcelXmlPath); 158 159 // start glbenchmark and wait for test to complete 160 isTimedOut = false; 161 long benchmarkStartTime = System.currentTimeMillis(); 162 163 device.executeShellCommand("am start -a android.intent.action.MAIN " 164 + "-n com.glbenchmark.glbenchmark25/com.glbenchmark.activities.MainActivity " 165 + "--ez custom true"); 166 isRunningBenchmark = true; 167 while (isRunningBenchmark && !isResultGenerated && !isTimedOut) { 168 RunUtil.getDefault().sleep(POLLING_INTERVAL_MS); 169 isTimedOut = (System.currentTimeMillis() - benchmarkStartTime >= TIMEOUT_MS); 170 isResultGenerated = device.doesFileExist(resultExcelXmlPath); 171 isRunningBenchmark = device.executeShellCommand("ps").contains("glbenchmark"); 172 } 173 174 if (isTimedOut) { 175 errMsg = "GLBenchmark timed out."; 176 } else { 177 // pull result from device 178 File benchmarkReport = device.pullFile(mGlbenchmarkResultXmlPath); 179 if (benchmarkReport != null) { 180 // parse result 181 CLog.i("== GLBenchmark result =="); 182 Map<String, String> benchmarkResult = parseResultXml(benchmarkReport); 183 if (benchmarkResult == null) { 184 errMsg = "Failed to parse GLBenchmark result XML."; 185 } else { 186 metrics = benchmarkResult; 187 } 188 // delete results from device and host 189 device.executeShellCommand(String.format("rm %s", mGlbenchmarkResultXmlPath)); 190 device.executeShellCommand(String.format("rm %s", resultExcelXmlPath)); 191 benchmarkReport.delete(); 192 } else { 193 errMsg = "GLBenchmark report not found."; 194 } 195 } 196 if (errMsg != null) { 197 CLog.e(errMsg); 198 listener.testFailed(testId, errMsg); 199 listener.testEnded(testId, TfMetricProtoUtil.upgradeConvert(metrics)); 200 listener.testRunFailed(errMsg); 201 } else { 202 long durationMs = System.currentTimeMillis() - testStartTime; 203 listener.testEnded(testId, TfMetricProtoUtil.upgradeConvert(metrics)); 204 listener.testRunEnded(durationMs, TfMetricProtoUtil.upgradeConvert(metrics)); 205 } 206 } 207 208 /** 209 * Parse GLBenchmark result XML. 210 * 211 * @param resultXml GLBenchmark result XML {@link File} 212 * @return a {@link HashMap} that contains metrics key and result 213 */ 214 private Map<String, String> parseResultXml(File resultXml) { 215 Map<String, String> benchmarkResult = new HashMap<String, String>(); 216 DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); 217 Document doc = null; 218 try { 219 DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); 220 doc = dBuilder.parse(resultXml); 221 } catch (ParserConfigurationException e) { 222 return null; 223 } catch (IOException e) { 224 return null; 225 } catch (SAXException e) { 226 return null; 227 } catch (IllegalArgumentException e) { 228 return null; 229 } 230 doc.getDocumentElement().normalize(); 231 232 NodeList nodes = doc.getElementsByTagName("test_result"); 233 for (int i = 0; i < nodes.getLength(); i++) { 234 Node node = nodes.item(i); 235 if (node.getNodeType() == Node.ELEMENT_NODE) { 236 Element testResult = (Element) node; 237 String testTitle = getData(testResult, "title"); 238 String testType = getData(testResult, "type"); 239 String fps = getData(testResult, "fps"); 240 String score = getData(testResult, "score"); 241 String testName = String.format("%s - %s", testTitle, testType); 242 if (METRICS_KEY_MAP.containsKey(testName)) { 243 if (testName.contains("Fill") || testName.contains("Triangle")) { 244 // Use Mtexels/sec or MTriangles/sec as unit 245 score = String.valueOf((long)(Double.parseDouble(score) / 1.0E6)); 246 } 247 CLog.i(String.format("%s: %s (fps=%s)", testName, score, fps)); 248 String testKey = METRICS_KEY_MAP.get(testName); 249 if (score != null && !score.trim().equals("0")) { 250 benchmarkResult.put(testKey, score); 251 if (fps != null && !fps.trim().equals("0.0")) { 252 try { 253 float fpsValue = Float.parseFloat(fps.replace("fps", "")); 254 benchmarkResult.put(testKey + "-fps", String.valueOf(fpsValue)); 255 } catch (NumberFormatException e) { 256 CLog.i(String.format("Got %s for fps value. Ignored.", fps)); 257 } 258 } 259 } 260 } 261 } 262 } 263 return benchmarkResult; 264 } 265 266 /** 267 * Get value in the first matching tag under the element 268 * 269 * @param element the parent {@link Element} of the tag 270 * @param tag {@link String} of the tag name 271 * @return a {@link String} that contains the value in the tag; returns null if not found. 272 */ 273 private String getData(Element element, String tag) { 274 NodeList tagNodes = element.getElementsByTagName(tag); 275 if (tagNodes.getLength() > 0) { 276 Node tagNode = tagNodes.item(0); 277 if (tagNode.getNodeType() == Node.ELEMENT_NODE) { 278 Node node = tagNode.getChildNodes().item(0); 279 if (node != null) { 280 return node.getNodeValue(); 281 } else { 282 return null; 283 } 284 } 285 } 286 return null; 287 } 288 } 289
