1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.framework.tests; 18 19 import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; 20 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; 21 import com.android.framework.tests.BandwidthStats.CompareResult; 22 import com.android.framework.tests.BandwidthStats.ComparisonRecord; 23 import com.android.tradefed.config.Option; 24 import com.android.tradefed.device.DeviceNotAvailableException; 25 import com.android.tradefed.device.ITestDevice; 26 import com.android.tradefed.log.LogUtil.CLog; 27 import com.android.tradefed.result.CollectingTestListener; 28 import com.android.tradefed.result.FileInputStreamSource; 29 import com.android.tradefed.result.ITestInvocationListener; 30 import com.android.tradefed.result.InputStreamSource; 31 import com.android.tradefed.result.LogDataType; 32 import com.android.tradefed.result.TestResult; 33 import com.android.tradefed.testtype.IDeviceTest; 34 import com.android.tradefed.testtype.IRemoteTest; 35 import com.android.tradefed.util.FileUtil; 36 import com.android.tradefed.util.IRunUtil.IRunnableResult; 37 import com.android.tradefed.util.MultiMap; 38 import com.android.tradefed.util.RunUtil; 39 import com.android.tradefed.util.StreamUtil; 40 import com.android.tradefed.util.net.HttpHelper; 41 import com.android.tradefed.util.net.IHttpHelper; 42 import com.android.tradefed.util.net.IHttpHelper.DataSizeException; 43 import com.android.tradefed.util.proto.TfMetricProtoUtil; 44 45 import org.junit.Assert; 46 47 import java.io.BufferedWriter; 48 import java.io.File; 49 import java.io.FileNotFoundException; 50 import java.io.FileOutputStream; 51 import java.io.IOException; 52 import java.io.OutputStreamWriter; 53 import java.util.Collection; 54 import java.util.HashMap; 55 import java.util.Map; 56 57 /** 58 * Test that instruments a bandwidth test, gathers bandwidth metrics, and posts 59 * the results to the Release Dashboard. 60 */ 61 public class BandwidthMicroBenchMarkTest implements IDeviceTest, IRemoteTest { 62 63 ITestDevice mTestDevice = null; 64 65 @Option(name = "test-package-name", description = "Android test package name.") 66 private String mTestPackageName; 67 68 @Option(name = "test-class-name", description = "Test class name.") 69 private String mTestClassName; 70 71 @Option(name = "test-method-name", description = "Test method name.") 72 private String mTestMethodName; 73 74 @Option(name = "test-label", 75 description = "Test label to identify the test run.") 76 private String mTestLabel; 77 78 @Option(name = "bandwidth-test-server", 79 description = "Test label to use when posting to dashboard.", 80 importance=Option.Importance.IF_UNSET) 81 private String mTestServer; 82 83 @Option(name = "ssid", 84 description = "The ssid to use for the wifi connection.") 85 private String mSsid; 86 87 @Option(name = "initial-server-poll-interval-ms", 88 description = "The initial poll interval in msecs for querying the test server.") 89 private int mInitialPollIntervalMs = 1 * 1000; 90 91 @Option(name = "server-total-timeout-ms", 92 description = "The total timeout in msecs for querying the test server.") 93 private int mTotalTimeoutMs = 40 * 60 * 1000; 94 95 @Option(name = "server-query-op-timeout-ms", 96 description = "The timeout in msecs for a single operation to query the test server.") 97 private int mQueryOpTimeoutMs = 2 * 60 * 1000; 98 99 @Option(name="difference-threshold", 100 description="The maximum allowed difference between network stats in percent") 101 private int mDifferenceThreshold = 5; 102 103 @Option(name="server-difference-threshold", 104 description="The maximum difference between the stats reported by the " + 105 "server and the device in percent") 106 private int mServerDifferenceThreshold = 6; 107 108 @Option(name = "compact-ru-key", 109 description = "Name of the reporting unit for pass/fail results") 110 private String mCompactRuKey; 111 112 @Option(name = "iface", description="Network interface on the device to use for stats", 113 importance = Option.Importance.ALWAYS) 114 private String mIface; 115 116 private static final String TEST_RUNNER = "com.android.bandwidthtest.BandwidthTestRunner"; 117 private static final String TEST_SERVER_QUERY = "query"; 118 private static final String DEVICE_ID_LABEL = "device_id"; 119 private static final String TIMESTAMP_LABEL = "timestamp"; 120 121 122 @Override 123 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 124 Assert.assertNotNull(mTestDevice); 125 126 Assert.assertNotNull("Need a test server, specify it using --bandwidth-test-server", 127 mTestServer); 128 129 // Run test 130 IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(mTestPackageName, 131 TEST_RUNNER, mTestDevice.getIDevice()); 132 runner.setMethodName(mTestClassName, mTestMethodName); 133 if (mSsid != null) { 134 runner.addInstrumentationArg("ssid", mSsid); 135 } 136 runner.addInstrumentationArg("server", mTestServer); 137 138 CollectingTestListener collectingListener = new CollectingTestListener(); 139 Assert.assertTrue( 140 mTestDevice.runInstrumentationTests(runner, collectingListener, listener)); 141 142 // Collect bandwidth metrics from the instrumentation test out. 143 Map<String, String> bandwidthTestMetrics = new HashMap<String, String>(); 144 Collection<TestResult> testResults = 145 collectingListener.getCurrentRunResults().getTestResults().values(); 146 if (testResults != null && testResults.iterator().hasNext()) { 147 Map<String, String> testMetrics = testResults.iterator().next().getMetrics(); 148 if (testMetrics != null) { 149 bandwidthTestMetrics.putAll(testMetrics); 150 } 151 } 152 153 // Fetch the data from the test server. 154 String deviceId = bandwidthTestMetrics.get(DEVICE_ID_LABEL); 155 String timestamp = bandwidthTestMetrics.get(TIMESTAMP_LABEL); 156 Assert.assertNotNull("Failed to fetch deviceId from server", deviceId); 157 Assert.assertNotNull("Failed to fetch timestamp from server", timestamp); 158 Map<String, String> serverData = fetchDataFromTestServer(deviceId, timestamp); 159 160 // Calculate additional network sanity stats - pre-framework logic network stats 161 BandwidthUtils bw = new BandwidthUtils(mTestDevice, mIface); 162 reportPassFail(listener, mCompactRuKey, bw, serverData, bandwidthTestMetrics); 163 164 saveFile("/proc/net/dev", "proc_net_dev", listener); 165 saveFile("/proc/net/xt_qtaguid/stats", "qtaguid_stats", listener); 166 167 } 168 169 private void saveFile(String remoteFilename, String spongeName, 170 ITestInvocationListener listener) throws DeviceNotAvailableException { 171 File f = mTestDevice.pullFile(remoteFilename); 172 if (f == null) { 173 CLog.w("Failed to pull %s", remoteFilename); 174 return; 175 } 176 177 saveFile(spongeName, listener, f); 178 } 179 180 private void saveFile(String spongeName, ITestInvocationListener listener, File file) { 181 try (InputStreamSource stream = new FileInputStreamSource(file)) { 182 listener.testLog(spongeName, LogDataType.TEXT, stream); 183 } 184 } 185 186 /** 187 * Fetch the bandwidth test data recorded on the test server. 188 * 189 * @param deviceId 190 * @param timestamp 191 * @return a map of the data that was recorded by the test server. 192 */ 193 private Map<String, String> fetchDataFromTestServer(String deviceId, String timestamp) { 194 IHttpHelper httphelper = new HttpHelper(); 195 MultiMap<String,String> params = new MultiMap<String,String> (); 196 params.put("device_id", deviceId); 197 params.put("timestamp", timestamp); 198 String queryUrl = mTestServer; 199 if (!queryUrl.endsWith("/")) { 200 queryUrl += "/"; 201 } 202 queryUrl += TEST_SERVER_QUERY; 203 QueryRunnable runnable = new QueryRunnable(httphelper, queryUrl, params); 204 if (RunUtil.getDefault().runEscalatingTimedRetry(mQueryOpTimeoutMs, mInitialPollIntervalMs, 205 mQueryOpTimeoutMs, mTotalTimeoutMs, runnable)) { 206 return runnable.getServerResponse(); 207 } else { 208 CLog.w("Failed to query test server", runnable.getException()); 209 } 210 return null; 211 } 212 213 private static class QueryRunnable implements IRunnableResult { 214 private final IHttpHelper mHttpHelper; 215 private final String mBaseUrl; 216 private final MultiMap<String,String> mParams; 217 private Map<String, String> mServerResponse = null; 218 private Exception mException = null; 219 220 public QueryRunnable(IHttpHelper helper, String testServerUrl, 221 MultiMap<String,String> params) { 222 mHttpHelper = helper; 223 mBaseUrl = testServerUrl; 224 mParams = params; 225 } 226 227 /** 228 * Perform a single bandwidth test server query, storing the response or 229 * the associated exception in case of error. 230 */ 231 @Override 232 public boolean run() { 233 try { 234 String serverResponse = mHttpHelper.doGet(mHttpHelper.buildUrl(mBaseUrl, mParams)); 235 mServerResponse = parseServerResponse(serverResponse); 236 return true; 237 } catch (IOException e) { 238 CLog.i("IOException %s when contacting test server", e.getMessage()); 239 mException = e; 240 } catch (DataSizeException e) { 241 CLog.i("Unexpected oversized response when contacting test server"); 242 mException = e; 243 } 244 return false; 245 } 246 247 /** 248 * Returns exception. 249 * 250 * @return the last {@link Exception} that occurred when performing 251 * run(). 252 */ 253 public Exception getException() { 254 return mException; 255 } 256 257 /** 258 * Returns the server response. 259 * 260 * @return a map of the server response. 261 */ 262 public Map<String, String> getServerResponse() { 263 return mServerResponse; 264 } 265 266 /** 267 * {@inheritDoc} 268 */ 269 @Override 270 public void cancel() { 271 // ignore 272 } 273 } 274 275 /** 276 * Helper to parse test server's response into a map 277 * <p> 278 * Exposed for unit testing. 279 * 280 * @param serverResponse {@link String} for the test server http request 281 * @return a map representation of the server response 282 */ 283 public static Map<String, String> parseServerResponse(String serverResponse) { 284 // No such test run was recorded. 285 if (serverResponse == null || serverResponse.trim().length() == 0) { 286 return null; 287 } 288 final String[] responseLines = serverResponse.split("\n"); 289 Map<String, String> results = new HashMap<String, String>(); 290 for (String responseLine : responseLines) { 291 final String[] responsePairs = responseLine.split(" "); 292 for (String responsePair : responsePairs) { 293 final String[] pair = responsePair.split(":", 2); 294 if (pair.length >= 2) { 295 results.put(pair[0], pair[1]); 296 } else { 297 CLog.w("Invalid server response: %s", responsePair); 298 } 299 } 300 } 301 return results; 302 } 303 304 /** 305 * Fetch the last stats from event log and calculate the differences. 306 * 307 * @throws DeviceNotAvailableException 308 */ 309 private boolean evaluateEventLog(ITestInvocationListener listener) throws DeviceNotAvailableException { 310 // issue a force update of stats 311 String res = mTestDevice.executeShellCommand("dumpsys netstats poll"); 312 if (!res.contains("Forced poll")) { 313 CLog.w("Failed to force a poll on the device."); 314 } 315 // fetch events log 316 String log = mTestDevice.executeShellCommand("logcat -d -b events"); 317 if (log != null) { 318 return evaluateStats("netstats_mobile_sample", log, listener); 319 } 320 return false; 321 } 322 323 /** 324 * Parse a log output for a given key and calculate the network stats. 325 * 326 * @param key {@link String} to search for in the log 327 * @param log obtained from adb logcat -b events 328 * @param listener the {@link ITestInvocationListener} where to report results. 329 */ 330 private boolean evaluateStats(String key, String log, ITestInvocationListener listener) { 331 File filteredEventLog = null; 332 BufferedWriter out = null; 333 boolean passed = true; 334 335 try { 336 filteredEventLog = File.createTempFile(String.format("%s_event_log", key), ".txt"); 337 out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filteredEventLog))); 338 String[] parts = log.split("\n"); 339 for (int i = parts.length - 1; i > 0; i--) { 340 String str = parts[i]; 341 if (str.contains(key)) { 342 out.write(str); 343 passed = passed && evaluateEventLogLine(str); 344 } 345 } 346 out.flush(); 347 saveFile(key + "_event_log", listener, filteredEventLog); 348 return passed; 349 } catch (FileNotFoundException e) { 350 CLog.w("Could not create file to save event log: %s", e.getMessage()); 351 return false; 352 } catch (IOException e) { 353 CLog.w("Could not save event log file: %s", e.getMessage()); 354 } finally { 355 StreamUtil.close(out); 356 FileUtil.deleteFile(filteredEventLog); 357 } 358 return false; 359 } 360 361 private boolean evaluateEventLogLine(String line) { 362 int start = line.lastIndexOf("["); 363 int end = line.lastIndexOf("]"); 364 String subStr = line.substring(start + 1, end); 365 String[] statsStrArray = subStr.split(","); 366 if (statsStrArray.length != 13) { 367 CLog.e("Failed to parse for \"%s\" in log.", line); 368 return false; 369 } 370 long xtRb = Long.parseLong(statsStrArray[4].trim()); 371 long xtTb = Long.parseLong(statsStrArray[5].trim()); 372 long xtRp = Long.parseLong(statsStrArray[6].trim()); 373 long xtTp = Long.parseLong(statsStrArray[7].trim()); 374 long uidRb = Long.parseLong(statsStrArray[8].trim()); 375 long uidTb = Long.parseLong(statsStrArray[9].trim()); 376 long uidRp = Long.parseLong(statsStrArray[10].trim()); 377 long uidTp = Long.parseLong(statsStrArray[11].trim()); 378 379 BandwidthStats xtStats = new BandwidthStats(xtRb, xtRp, xtTb, xtTp); 380 BandwidthStats uidStats = new BandwidthStats(uidRb, uidRp, uidTb, uidTp); 381 boolean result = true; 382 CompareResult compareResult = xtStats.compareAll(uidStats, mDifferenceThreshold); 383 result &= compareResult.getResult(); 384 if (!compareResult.getResult()) { 385 CLog.i("Failure comparing netstats_mobile_sample xt and uid"); 386 printFailures(compareResult); 387 } 388 if (!result) { 389 CLog.i("Failed line: %s", line); 390 } 391 return result; 392 } 393 394 /** 395 * Compare the data reported by instrumentation to uid breakdown reported by the kernel, 396 * the sum of uid breakdown and the total reported by the kernel and the data reported by 397 * instrumentation to the data reported by the server. 398 * @param listener result reporter 399 * @param compactRuKey key to use when posting to rdb. 400 * @param utils data parsed from the kernel. 401 * @param instrumentationData data reported by the test. 402 * @param serverData data reported by the server. 403 * @throws DeviceNotAvailableException 404 */ 405 private void reportPassFail(ITestInvocationListener listener, String compactRuKey, 406 BandwidthUtils utils, Map<String, String> serverData, 407 Map<String, String> instrumentationData) throws DeviceNotAvailableException { 408 if (compactRuKey == null) return; 409 410 int passCount = 0; 411 int failCount = 0; 412 413 // Calculate the difference between what framework reports and what the kernel reports 414 boolean download = Boolean.parseBoolean(serverData.get("download")); 415 long frameworkUidBytes = 0; 416 if (download) { 417 frameworkUidBytes = Long.parseLong(instrumentationData.get("PROF_rx")); 418 } else { 419 frameworkUidBytes = Long.parseLong(instrumentationData.get("PROF_tx")); 420 } 421 422 // Compare data reported by the server and the instrumentation 423 long serverBytes = Long.parseLong(serverData.get("size")); 424 float diff = Math.abs(BandwidthStats.computePercentDifference( 425 serverBytes, frameworkUidBytes)); 426 if (diff < mServerDifferenceThreshold) { 427 passCount += 1; 428 } else { 429 CLog.i("Comparing between server and instrumentation failed expected %d got %d", 430 serverBytes, frameworkUidBytes); 431 failCount += 1; 432 } 433 434 if (evaluateEventLog(listener)) { 435 passCount += 1; 436 } else { 437 failCount += 1; 438 } 439 440 Map<String, String> postMetrics = new HashMap<String, String>(); 441 postMetrics.put("Pass", String.valueOf(passCount)); 442 postMetrics.put("Fail", String.valueOf(failCount)); 443 444 listener.testRunStarted(compactRuKey, 0); 445 listener.testRunEnded(0, TfMetricProtoUtil.upgradeConvert(postMetrics)); 446 } 447 448 private void printFailures(CompareResult result) { 449 for (ComparisonRecord failure : result.getFailures()) { 450 CLog.i(failure.toString()); 451 } 452 } 453 454 /** 455 * {@inheritDoc} 456 */ 457 @Override 458 public void setDevice(ITestDevice device) { 459 mTestDevice = device; 460 } 461 462 /** 463 * {@inheritDoc} 464 */ 465 @Override 466 public ITestDevice getDevice() { 467 return mTestDevice; 468 } 469 } 470
