1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.performance.tests; 17 18 import com.android.tradefed.config.Option; 19 import com.android.tradefed.config.Option.Importance; 20 import com.android.tradefed.device.DeviceNotAvailableException; 21 import com.android.tradefed.device.ITestDevice; 22 import com.android.tradefed.log.LogUtil.CLog; 23 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 24 import com.android.tradefed.result.ITestInvocationListener; 25 import com.android.tradefed.result.TestDescription; 26 import com.android.tradefed.testtype.IDeviceTest; 27 import com.android.tradefed.testtype.IRemoteTest; 28 import com.android.tradefed.util.AbiFormatter; 29 import com.android.tradefed.util.SimplePerfResult; 30 import com.android.tradefed.util.SimplePerfUtil; 31 import com.android.tradefed.util.SimplePerfUtil.SimplePerfType; 32 import com.android.tradefed.util.SimpleStats; 33 import com.android.tradefed.util.proto.TfMetricProtoUtil; 34 35 import org.junit.Assert; 36 37 import java.text.NumberFormat; 38 import java.text.ParseException; 39 import java.util.ArrayList; 40 import java.util.HashMap; 41 import java.util.List; 42 import java.util.Locale; 43 import java.util.Map; 44 import java.util.regex.Matcher; 45 import java.util.regex.Pattern; 46 47 /** 48 * This test is targeting eMMC performance on read/ write. 49 */ 50 public class EmmcPerformanceTest implements IDeviceTest, IRemoteTest { 51 private enum TestType { 52 DD, 53 RANDOM; 54 } 55 56 private static final String RUN_KEY = "emmc_performance_tests"; 57 58 private static final String SEQUENTIAL_READ_KEY = "sequential_read"; 59 private static final String SEQUENTIAL_WRITE_KEY = "sequential_write"; 60 private static final String RANDOM_READ_KEY = "random_read"; 61 private static final String RANDOM_WRITE_KEY = "random_write"; 62 private static final String PERF_RANDOM = "/data/local/tmp/rand_emmc_perf|#ABI32#|"; 63 64 private static final Pattern DD_PATTERN = Pattern.compile( 65 "\\d+ bytes transferred in \\d+\\.\\d+ secs \\((\\d+) bytes/sec\\)"); 66 67 private static final Pattern EMMC_RANDOM_PATTERN = Pattern.compile( 68 "(\\d+) (\\d+)byte iops/sec"); 69 private static final int BLOCK_SIZE = 1048576; 70 private static final int SEQ_COUNT = 200; 71 72 @Option(name = "cpufreq", description = "The path to the cpufreq directory on the DUT.") 73 private String mCpufreq = "/sys/devices/system/cpu/cpu0/cpufreq"; 74 75 @Option(name = "auto-discover-cache-info", 76 description = 77 "Indicate if test should attempt auto discover cache path and partition size " 78 + "from the test device. Default to be false, ie. manually set " 79 + "cache-device and cache-partition-size, or use default." 80 + " If fail to discover, it will fallback to what is set in " 81 + "cache-device") 82 private boolean mAutoDiscoverCacheInfo = false; 83 84 @Option(name = "cache-device", description = "The path to the cache block device on the DUT." + 85 " Nakasi: /dev/block/platform/sdhci-tegra.3/by-name/CAC\n" + 86 " Prime: /dev/block/platform/omap/omap_hsmmc.0/by-name/cache\n" + 87 " Stingray: /dev/block/platform/sdhci-tegra.3/by-name/cache\n" + 88 " Crespo: /dev/block/platform/s3c-sdhci.0/by-name/userdata\n", 89 importance = Importance.IF_UNSET) 90 private String mCache = null; 91 92 @Option(name = "iterations", description = "The number of iterations to run") 93 private int mIterations = 100; 94 95 @Option(name = AbiFormatter.FORCE_ABI_STRING, 96 description = AbiFormatter.FORCE_ABI_DESCRIPTION, 97 importance = Importance.IF_UNSET) 98 private String mForceAbi = null; 99 100 @Option(name = "cache-partition-size", description = "Cache partiton size in MB") 101 private static int mCachePartitionSize = 100; 102 103 @Option(name = "simpleperf-mode", 104 description = "Whether use simpleperf to get low level metrics") 105 private boolean mSimpleperfMode = false; 106 107 @Option(name = "simpleperf-argu", description = "simpleperf arguments") 108 private List<String> mSimpleperfArgu = new ArrayList<String>(); 109 110 ITestDevice mTestDevice = null; 111 SimplePerfUtil mSpUtil = null; 112 113 /** 114 * {@inheritDoc} 115 */ 116 @Override 117 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 118 try { 119 setUp(); 120 121 listener.testRunStarted(RUN_KEY, 5); 122 long beginTime = System.currentTimeMillis(); 123 Map<String, String> metrics = new HashMap<String, String>(); 124 125 runSequentialRead(mIterations, listener, metrics); 126 runSequentialWrite(mIterations, listener, metrics); 127 // FIXME: Figure out cache issues with random read and reenable test. 128 // runRandomRead(mIterations, listener, metrics); 129 // runRandomWrite(mIterations, listener, metrics); 130 131 CLog.d("Metrics: %s", metrics.toString()); 132 listener.testRunEnded( 133 (System.currentTimeMillis() - beginTime), 134 TfMetricProtoUtil.upgradeConvert(metrics)); 135 } finally { 136 cleanUp(); 137 } 138 } 139 140 /** 141 * Run the sequential read test. 142 */ 143 private void runSequentialRead(int iterations, ITestInvocationListener listener, 144 Map<String, String> metrics) throws DeviceNotAvailableException { 145 String command = String.format("dd if=%s of=/dev/null bs=%d count=%d", mCache, BLOCK_SIZE, 146 SEQ_COUNT); 147 runTest(SEQUENTIAL_READ_KEY, command, TestType.DD, true, iterations, listener, metrics); 148 } 149 150 /** 151 * Run the sequential write test. 152 */ 153 private void runSequentialWrite(int iterations, ITestInvocationListener listener, 154 Map<String, String> metrics) throws DeviceNotAvailableException { 155 String command = String.format("dd if=/dev/zero of=%s bs=%d count=%d", mCache, BLOCK_SIZE, 156 SEQ_COUNT); 157 runTest(SEQUENTIAL_WRITE_KEY, command, TestType.DD, false, iterations, listener, metrics); 158 } 159 160 /** 161 * Run the random read test. 162 */ 163 @SuppressWarnings("unused") 164 private void runRandomRead(int iterations, ITestInvocationListener listener, 165 Map<String, String> metrics) throws DeviceNotAvailableException { 166 String command = String.format("%s -r %d %s", 167 AbiFormatter.formatCmdForAbi(PERF_RANDOM, mForceAbi), mCachePartitionSize, mCache); 168 runTest(RANDOM_READ_KEY, command, TestType.RANDOM, true, iterations, listener, metrics); 169 } 170 171 /** 172 * Run the random write test with OSYNC disabled. 173 */ 174 private void runRandomWrite(int iterations, ITestInvocationListener listener, 175 Map<String, String> metrics) throws DeviceNotAvailableException { 176 String command = String.format("%s -w %d %s", 177 AbiFormatter.formatCmdForAbi(PERF_RANDOM, mForceAbi), mCachePartitionSize, mCache); 178 runTest(RANDOM_WRITE_KEY, command, TestType.RANDOM, false, iterations, listener, metrics); 179 } 180 181 /** 182 * Run a test for a number of iterations. 183 * 184 * @param testKey the key used to report metrics. 185 * @param command the command to be run on the device. 186 * @param type the {@link TestType}, which determines how each iteration should be run. 187 * @param dropCache whether to drop the cache before starting each iteration. 188 * @param iterations the number of iterations to run. 189 * @param listener the {@link ITestInvocationListener}. 190 * @param metrics the map to store metrics of. 191 * @throws DeviceNotAvailableException If the device was not available. 192 */ 193 private void runTest(String testKey, String command, TestType type, boolean dropCache, 194 int iterations, ITestInvocationListener listener, Map<String, String> metrics) 195 throws DeviceNotAvailableException { 196 CLog.i("Starting test %s", testKey); 197 198 TestDescription id = new TestDescription(RUN_KEY, testKey); 199 listener.testStarted(id); 200 201 Map<String, SimpleStats> simpleperfMetricsMap = new HashMap<String, SimpleStats>(); 202 SimpleStats stats = new SimpleStats(); 203 for (int i = 0; i < iterations; i++) { 204 if (dropCache) { 205 dropCache(); 206 } 207 208 Double kbps = null; 209 switch (type) { 210 case DD: 211 kbps = runDdIteration(command, simpleperfMetricsMap); 212 break; 213 case RANDOM: 214 kbps = runRandomIteration(command, simpleperfMetricsMap); 215 break; 216 } 217 218 if (kbps != null) { 219 CLog.i("Result for %s, iteration %d: %f KBps", testKey, i + 1, kbps); 220 stats.add(kbps); 221 } else { 222 CLog.w("Skipping %s, iteration %d", testKey, i + 1); 223 } 224 } 225 226 if (stats.mean() != null) { 227 metrics.put(testKey, Double.toString(stats.median())); 228 for (Map.Entry<String, SimpleStats> entry : simpleperfMetricsMap.entrySet()) { 229 metrics.put(String.format("%s_%s", testKey, entry.getKey()), 230 Double.toString(entry.getValue().median())); 231 } 232 } else { 233 listener.testFailed(id, "No metrics to report (see log)"); 234 } 235 CLog.i("Test %s finished: mean=%f, stdev=%f, samples=%d", testKey, stats.mean(), 236 stats.stdev(), stats.size()); 237 listener.testEnded(id, new HashMap<String, Metric>()); 238 } 239 240 /** 241 * Run a single iteration of the dd (sequential) test. 242 * 243 * @param command the command to run on the device. 244 * @param simpleperfMetricsMap the map contain simpleperf metrics aggregated results 245 * @return The speed of the test in KBps or null if there was an error running or parsing the 246 * test. 247 * @throws DeviceNotAvailableException If the device was not available. 248 */ 249 private Double runDdIteration(String command, Map<String, SimpleStats> simpleperfMetricsMap) 250 throws DeviceNotAvailableException { 251 String[] output; 252 SimplePerfResult spResult = null; 253 if (mSimpleperfMode) { 254 spResult = mSpUtil.executeCommand(command); 255 output = spResult.getCommandRawOutput().split("\n"); 256 } else { 257 output = mTestDevice.executeShellCommand(command).split("\n"); 258 } 259 String line = output[output.length - 1].trim(); 260 261 Matcher m = DD_PATTERN.matcher(line); 262 if (m.matches()) { 263 simpleperfResultAggregation(spResult, simpleperfMetricsMap); 264 return convertBpsToKBps(Double.parseDouble(m.group(1))); 265 } else { 266 CLog.w("Line \"%s\" did not match expected output, ignoring", line); 267 return null; 268 } 269 } 270 271 /** 272 * Run a single iteration of the random test. 273 * 274 * @param command the command to run on the device. 275 * @param simpleperfMetricsMap the map contain simpleperf metrics aggregated results 276 * @return The speed of the test in KBps or null if there was an error running or parsing the 277 * test. 278 * @throws DeviceNotAvailableException If the device was not available. 279 */ 280 private Double runRandomIteration(String command, Map<String, SimpleStats> simpleperfMetricsMap) 281 throws DeviceNotAvailableException { 282 String output; 283 SimplePerfResult spResult = null; 284 if (mSimpleperfMode) { 285 spResult = mSpUtil.executeCommand(command); 286 output = spResult.getCommandRawOutput(); 287 } else { 288 output = mTestDevice.executeShellCommand(command); 289 } 290 Matcher m = EMMC_RANDOM_PATTERN.matcher(output.trim()); 291 if (m.matches()) { 292 simpleperfResultAggregation(spResult, simpleperfMetricsMap); 293 return convertIopsToKBps(Double.parseDouble(m.group(1))); 294 } else { 295 CLog.w("Line \"%s\" did not match expected output, ignoring", output); 296 return null; 297 } 298 } 299 300 /** 301 * Helper function to aggregate simpleperf results 302 * 303 * @param spResult object that holds simpleperf results 304 * @param simpleperfMetricsMap map holds aggregated simpleperf results 305 */ 306 private void simpleperfResultAggregation(SimplePerfResult spResult, 307 Map<String, SimpleStats> simpleperfMetricsMap) { 308 if (mSimpleperfMode) { 309 Assert.assertNotNull("simpleperf result is null object", spResult); 310 for (Map.Entry<String, String> entry : spResult.getBenchmarkMetrics().entrySet()) { 311 try { 312 Double metricValue = NumberFormat.getNumberInstance(Locale.US) 313 .parse(entry.getValue()).doubleValue(); 314 if (!simpleperfMetricsMap.containsKey(entry.getKey())) { 315 SimpleStats newStat = new SimpleStats(); 316 simpleperfMetricsMap.put(entry.getKey(), newStat); 317 } 318 simpleperfMetricsMap.get(entry.getKey()).add(metricValue); 319 } catch (ParseException e) { 320 CLog.e("Simpleperf metrics parse failure: " + e.toString()); 321 } 322 } 323 } 324 } 325 326 /** 327 * Drop the disk cache on the device. 328 */ 329 private void dropCache() throws DeviceNotAvailableException { 330 mTestDevice.executeShellCommand("echo 3 > /proc/sys/vm/drop_caches"); 331 } 332 333 /** 334 * Convert bytes / sec reported by the dd tests into KBps. 335 */ 336 private double convertBpsToKBps(double bps) { 337 return bps / 1024; 338 } 339 340 /** 341 * Convert the iops reported by the random tests into KBps. 342 * <p> 343 * The iops is number of 4kB block reads/writes per sec. This makes the conversion factor 4. 344 * </p> 345 */ 346 private double convertIopsToKBps(double iops) { 347 return 4 * iops; 348 } 349 350 /** 351 * Setup the device for tests by unmounting partitions and maxing the cpu speed. 352 */ 353 private void setUp() throws DeviceNotAvailableException { 354 if (mAutoDiscoverCacheInfo) { 355 discoverCacheInfo(); 356 } 357 mTestDevice.executeShellCommand("umount /sdcard"); 358 mTestDevice.executeShellCommand("umount /data"); 359 mTestDevice.executeShellCommand("umount /cache"); 360 361 mTestDevice.executeShellCommand( 362 String.format("cat %s/cpuinfo_max_freq > %s/scaling_max_freq", mCpufreq, mCpufreq)); 363 mTestDevice.executeShellCommand( 364 String.format("cat %s/cpuinfo_max_freq > %s/scaling_min_freq", mCpufreq, mCpufreq)); 365 366 if (mSimpleperfMode) { 367 mSpUtil = SimplePerfUtil.newInstance(mTestDevice, SimplePerfType.STAT); 368 if (mSimpleperfArgu.size() == 0) { 369 mSimpleperfArgu.add("-e cpu-cycles:k,cpu-cycles:u"); 370 } 371 mSpUtil.setArgumentList(mSimpleperfArgu); 372 } 373 } 374 375 /** 376 * Attempt to detect cache path and cache partition size automatically 377 */ 378 private void discoverCacheInfo() throws DeviceNotAvailableException { 379 // Expected output look similar to the following: 380 // 381 // > ... vdc dump | grep cache 382 // 0 4123 /dev/block/platform/soc/7824900.sdhci/by-name/cache /cache ext4 rw, \ 383 // seclabel,nosuid,nodev,noatime,discard,data=ordered 0 0 384 if (mTestDevice.enableAdbRoot()) { 385 String output = mTestDevice.executeShellCommand("vdc dump | grep cache"); 386 CLog.d("Output from shell command 'vdc dump | grep cache':\n%s", output); 387 String[] segments = output.split("\\s+"); 388 if (segments.length >= 3) { 389 mCache = segments[2]; 390 } else { 391 CLog.w("Fail to detect cache path. Fall back to use '%s'", mCache); 392 } 393 } else { 394 CLog.d("Cannot get cache path because device %s is not rooted.", 395 mTestDevice.getSerialNumber()); 396 } 397 398 // Expected output looks similar to the following: 399 // 400 // > ... df cache 401 // Filesystem 1K-blocks Used Available Use% Mounted on 402 // /dev/block/mmcblk0p34 60400 56 60344 1% /cache 403 String output = mTestDevice.executeShellCommand("df cache"); 404 CLog.d(String.format("Output from shell command 'df cache':\n%s", output)); 405 String[] lines = output.split("\r?\n"); 406 if (lines.length >= 2) { 407 String[] segments = lines[1].split("\\s+"); 408 if (segments.length >= 2) { 409 if (lines[0].toLowerCase().contains("1k-blocks")) { 410 mCachePartitionSize = Integer.parseInt(segments[1]) / 1024; 411 } else { 412 throw new IllegalArgumentException("Unknown unit for the cache size."); 413 } 414 } 415 } 416 417 CLog.d("cache-device is set to %s ...", mCache); 418 CLog.d("cache-partition-size is set to %d ...", mCachePartitionSize); 419 } 420 421 /** 422 * Clean up the device by formatting a new cache partition. 423 */ 424 private void cleanUp() throws DeviceNotAvailableException { 425 mTestDevice.executeShellCommand(String.format("mke2fs %s", mCache)); 426 } 427 428 /** 429 * {@inheritDoc} 430 */ 431 @Override 432 public void setDevice(ITestDevice device) { 433 mTestDevice = device; 434 } 435 436 /** 437 * {@inheritDoc} 438 */ 439 @Override 440 public ITestDevice getDevice() { 441 return mTestDevice; 442 } 443 } 444 445