1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.performance.tests; 17 18 import com.android.ddmlib.testrunner.TestIdentifier; 19 import com.android.tradefed.config.Option; 20 import com.android.tradefed.config.Option.Importance; 21 import com.android.tradefed.device.DeviceNotAvailableException; 22 import com.android.tradefed.device.ITestDevice; 23 import com.android.tradefed.log.LogUtil.CLog; 24 import com.android.tradefed.result.ITestInvocationListener; 25 import com.android.tradefed.testtype.IDeviceTest; 26 import com.android.tradefed.testtype.IRemoteTest; 27 import com.android.tradefed.util.AbiFormatter; 28 import com.android.tradefed.util.SimplePerfResult; 29 import com.android.tradefed.util.SimplePerfUtil; 30 import com.android.tradefed.util.SimplePerfUtil.SimplePerfType; 31 import com.android.tradefed.util.SimpleStats; 32 33 import org.junit.Assert; 34 35 import java.text.NumberFormat; 36 import java.text.ParseException; 37 import java.util.ArrayList; 38 import java.util.Collections; 39 import java.util.HashMap; 40 import java.util.List; 41 import java.util.Locale; 42 import java.util.Map; 43 import java.util.regex.Matcher; 44 import java.util.regex.Pattern; 45 46 /** 47 * This test is targeting eMMC performance on read/ write. 48 */ 49 public class EmmcPerformanceTest implements IDeviceTest, IRemoteTest { 50 private enum TestType { 51 DD, 52 RANDOM; 53 } 54 55 private static final String RUN_KEY = "emmc_performance_tests"; 56 57 private static final String SEQUENTIAL_READ_KEY = "sequential_read"; 58 private static final String SEQUENTIAL_WRITE_KEY = "sequential_write"; 59 private static final String RANDOM_READ_KEY = "random_read"; 60 private static final String RANDOM_WRITE_KEY = "random_write"; 61 private static final String PERF_RANDOM = "/data/local/tmp/rand_emmc_perf|#ABI32#|"; 62 63 private static final Pattern DD_PATTERN = Pattern.compile( 64 "\\d+ bytes transferred in \\d+\\.\\d+ secs \\((\\d+) bytes/sec\\)"); 65 66 private static final Pattern EMMC_RANDOM_PATTERN = Pattern.compile( 67 "(\\d+) (\\d+)byte iops/sec"); 68 private static final int BLOCK_SIZE = 1048576; 69 private static final int SEQ_COUNT = 200; 70 71 @Option(name = "cpufreq", description = "The path to the cpufreq directory on the DUT.") 72 private String mCpufreq = "/sys/devices/system/cpu/cpu0/cpufreq"; 73 74 @Option(name = "auto-discover-cache-info", 75 description = 76 "Indicate if test should attempt auto discover cache path and partition size " 77 + "from the test device. Default to be false, ie. manually set " 78 + "cache-device and cache-partition-size, or use default." 79 + " If fail to discover, it will fallback to what is set in " 80 + "cache-device") 81 private boolean mAutoDiscoverCacheInfo = false; 82 83 @Option(name = "cache-device", description = "The path to the cache block device on the DUT." + 84 " Nakasi: /dev/block/platform/sdhci-tegra.3/by-name/CAC\n" + 85 " Prime: /dev/block/platform/omap/omap_hsmmc.0/by-name/cache\n" + 86 " Stingray: /dev/block/platform/sdhci-tegra.3/by-name/cache\n" + 87 " Crespo: /dev/block/platform/s3c-sdhci.0/by-name/userdata\n", 88 importance = Importance.IF_UNSET) 89 private String mCache = null; 90 91 @Option(name = "iterations", description = "The number of iterations to run") 92 private int mIterations = 100; 93 94 @Option(name = AbiFormatter.FORCE_ABI_STRING, 95 description = AbiFormatter.FORCE_ABI_DESCRIPTION, 96 importance = Importance.IF_UNSET) 97 private String mForceAbi = null; 98 99 @Option(name = "cache-partition-size", description = "Cache partiton size in MB") 100 private static int mCachePartitionSize = 100; 101 102 @Option(name = "simpleperf-mode", 103 description = "Whether use simpleperf to get low level metrics") 104 private boolean mSimpleperfMode = false; 105 106 @Option(name = "simpleperf-argu", description = "simpleperf arguments") 107 private List<String> mSimpleperfArgu = new ArrayList<String>(); 108 109 ITestDevice mTestDevice = null; 110 SimplePerfUtil mSpUtil = null; 111 112 /** 113 * {@inheritDoc} 114 */ 115 @Override 116 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 117 try { 118 setUp(); 119 120 listener.testRunStarted(RUN_KEY, 5); 121 long beginTime = System.currentTimeMillis(); 122 Map<String, String> metrics = new HashMap<String, String>(); 123 124 runSequentialRead(mIterations, listener, metrics); 125 runSequentialWrite(mIterations, listener, metrics); 126 // FIXME: Figure out cache issues with random read and reenable test. 127 // runRandomRead(mIterations, listener, metrics); 128 runRandomWrite(mIterations, listener, metrics); 129 130 CLog.d("Metrics: %s", metrics.toString()); 131 listener.testRunEnded((System.currentTimeMillis() - beginTime), metrics); 132 } finally { 133 cleanUp(); 134 } 135 } 136 137 /** 138 * Run the sequential read test. 139 */ 140 private void runSequentialRead(int iterations, ITestInvocationListener listener, 141 Map<String, String> metrics) throws DeviceNotAvailableException { 142 String command = String.format("dd if=%s of=/dev/null bs=%d count=%d", mCache, BLOCK_SIZE, 143 SEQ_COUNT); 144 runTest(SEQUENTIAL_READ_KEY, command, TestType.DD, true, iterations, listener, metrics); 145 } 146 147 /** 148 * Run the sequential write test. 149 */ 150 private void runSequentialWrite(int iterations, ITestInvocationListener listener, 151 Map<String, String> metrics) throws DeviceNotAvailableException { 152 String command = String.format("dd if=/dev/zero of=%s bs=%d count=%d", mCache, BLOCK_SIZE, 153 SEQ_COUNT); 154 runTest(SEQUENTIAL_WRITE_KEY, command, TestType.DD, false, iterations, listener, metrics); 155 } 156 157 /** 158 * Run the random read test. 159 */ 160 @SuppressWarnings("unused") 161 private void runRandomRead(int iterations, ITestInvocationListener listener, 162 Map<String, String> metrics) throws DeviceNotAvailableException { 163 String command = String.format("%s -r %d %s", 164 AbiFormatter.formatCmdForAbi(PERF_RANDOM, mForceAbi), mCachePartitionSize, mCache); 165 runTest(RANDOM_READ_KEY, command, TestType.RANDOM, true, iterations, listener, metrics); 166 } 167 168 /** 169 * Run the random write test with OSYNC disabled. 170 */ 171 private void runRandomWrite(int iterations, ITestInvocationListener listener, 172 Map<String, String> metrics) throws DeviceNotAvailableException { 173 String command = String.format("%s -w %d %s", 174 AbiFormatter.formatCmdForAbi(PERF_RANDOM, mForceAbi), mCachePartitionSize, mCache); 175 runTest(RANDOM_WRITE_KEY, command, TestType.RANDOM, false, iterations, listener, metrics); 176 } 177 178 /** 179 * Run a test for a number of iterations. 180 * 181 * @param testKey the key used to report metrics. 182 * @param command the command to be run on the device. 183 * @param type the {@link TestType}, which determines how each iteration should be run. 184 * @param dropCache whether to drop the cache before starting each iteration. 185 * @param iterations the number of iterations to run. 186 * @param listener the {@link ITestInvocationListener}. 187 * @param metrics the map to store metrics of. 188 * @throws DeviceNotAvailableException If the device was not available. 189 */ 190 private void runTest(String testKey, String command, TestType type, boolean dropCache, 191 int iterations, ITestInvocationListener listener, Map<String, String> metrics) 192 throws DeviceNotAvailableException { 193 CLog.i("Starting test %s", testKey); 194 195 TestIdentifier id = new TestIdentifier(RUN_KEY, testKey); 196 listener.testStarted(id); 197 198 Map<String, SimpleStats> simpleperfMetricsMap = new HashMap<String, SimpleStats>(); 199 SimpleStats stats = new SimpleStats(); 200 for (int i = 0; i < iterations; i++) { 201 if (dropCache) { 202 dropCache(); 203 } 204 205 Double kbps = null; 206 switch (type) { 207 case DD: 208 kbps = runDdIteration(command, simpleperfMetricsMap); 209 break; 210 case RANDOM: 211 kbps = runRandomIteration(command, simpleperfMetricsMap); 212 break; 213 } 214 215 if (kbps != null) { 216 CLog.i("Result for %s, iteration %d: %f KBps", testKey, i + 1, kbps); 217 stats.add(kbps); 218 } else { 219 CLog.w("Skipping %s, iteration %d", testKey, i + 1); 220 } 221 } 222 223 if (stats.mean() != null) { 224 metrics.put(testKey, Double.toString(stats.median())); 225 for (Map.Entry<String, SimpleStats> entry : simpleperfMetricsMap.entrySet()) { 226 metrics.put(String.format("%s_%s", testKey, entry.getKey()), 227 Double.toString(entry.getValue().median())); 228 } 229 } else { 230 listener.testFailed(id, "No metrics to report (see log)"); 231 } 232 CLog.i("Test %s finished: mean=%f, stdev=%f, samples=%d", testKey, stats.mean(), 233 stats.stdev(), stats.size()); 234 Map<String, String> emptyMap = Collections.emptyMap(); 235 listener.testEnded(id, emptyMap); 236 } 237 238 /** 239 * Run a single iteration of the dd (sequential) test. 240 * 241 * @param command the command to run on the device. 242 * @param simpleperfMetricsMap the map contain simpleperf metrics aggregated results 243 * @return The speed of the test in KBps or null if there was an error running or parsing the 244 * test. 245 * @throws DeviceNotAvailableException If the device was not available. 246 */ 247 private Double runDdIteration(String command, Map<String, SimpleStats> simpleperfMetricsMap) 248 throws DeviceNotAvailableException { 249 String[] output; 250 SimplePerfResult spResult = null; 251 if (mSimpleperfMode) { 252 spResult = mSpUtil.executeCommand(command); 253 output = spResult.getCommandRawOutput().split("\n"); 254 } else { 255 output = mTestDevice.executeShellCommand(command).split("\n"); 256 } 257 String line = output[output.length - 1].trim(); 258 259 Matcher m = DD_PATTERN.matcher(line); 260 if (m.matches()) { 261 simpleperfResultAggregation(spResult, simpleperfMetricsMap); 262 return convertBpsToKBps(Double.parseDouble(m.group(1))); 263 } else { 264 CLog.w("Line \"%s\" did not match expected output, ignoring", line); 265 return null; 266 } 267 } 268 269 /** 270 * Run a single iteration of the random test. 271 * 272 * @param command the command to run on the device. 273 * @param simpleperfMetricsMap the map contain simpleperf metrics aggregated results 274 * @return The speed of the test in KBps or null if there was an error running or parsing the 275 * test. 276 * @throws DeviceNotAvailableException If the device was not available. 277 */ 278 private Double runRandomIteration(String command, Map<String, SimpleStats> simpleperfMetricsMap) 279 throws DeviceNotAvailableException { 280 String output; 281 SimplePerfResult spResult = null; 282 if (mSimpleperfMode) { 283 spResult = mSpUtil.executeCommand(command); 284 output = spResult.getCommandRawOutput(); 285 } else { 286 output = mTestDevice.executeShellCommand(command); 287 } 288 Matcher m = EMMC_RANDOM_PATTERN.matcher(output.trim()); 289 if (m.matches()) { 290 simpleperfResultAggregation(spResult, simpleperfMetricsMap); 291 return convertIopsToKBps(Double.parseDouble(m.group(1))); 292 } else { 293 CLog.w("Line \"%s\" did not match expected output, ignoring", output); 294 return null; 295 } 296 } 297 298 /** 299 * Helper function to aggregate simpleperf results 300 * 301 * @param spResult object that holds simpleperf results 302 * @param simpleperfMetricsMap map holds aggregated simpleperf results 303 */ 304 private void simpleperfResultAggregation(SimplePerfResult spResult, 305 Map<String, SimpleStats> simpleperfMetricsMap) { 306 if (mSimpleperfMode) { 307 Assert.assertNotNull("simpleperf result is null object", spResult); 308 for (Map.Entry<String, String> entry : spResult.getBenchmarkMetrics().entrySet()) { 309 try { 310 Double metricValue = NumberFormat.getNumberInstance(Locale.US) 311 .parse(entry.getValue()).doubleValue(); 312 if (!simpleperfMetricsMap.containsKey(entry.getKey())) { 313 SimpleStats newStat = new SimpleStats(); 314 simpleperfMetricsMap.put(entry.getKey(), newStat); 315 } 316 simpleperfMetricsMap.get(entry.getKey()).add(metricValue); 317 } catch (ParseException e) { 318 CLog.e("Simpleperf metrics parse failure: " + e.toString()); 319 } 320 } 321 } 322 } 323 324 /** 325 * Drop the disk cache on the device. 326 */ 327 private void dropCache() throws DeviceNotAvailableException { 328 mTestDevice.executeShellCommand("echo 3 > /proc/sys/vm/drop_caches"); 329 } 330 331 /** 332 * Convert bytes / sec reported by the dd tests into KBps. 333 */ 334 private double convertBpsToKBps(double bps) { 335 return bps / 1024; 336 } 337 338 /** 339 * Convert the iops reported by the random tests into KBps. 340 * <p> 341 * The iops is number of 4kB block reads/writes per sec. This makes the conversion factor 4. 342 * </p> 343 */ 344 private double convertIopsToKBps(double iops) { 345 return 4 * iops; 346 } 347 348 /** 349 * Setup the device for tests by unmounting partitions and maxing the cpu speed. 350 */ 351 private void setUp() throws DeviceNotAvailableException { 352 mTestDevice.executeShellCommand("umount /sdcard"); 353 mTestDevice.executeShellCommand("umount /data"); 354 mTestDevice.executeShellCommand("umount /cache"); 355 356 mTestDevice.executeShellCommand( 357 String.format("cat %s/cpuinfo_max_freq > %s/scaling_max_freq", mCpufreq, mCpufreq)); 358 mTestDevice.executeShellCommand( 359 String.format("cat %s/cpuinfo_max_freq > %s/scaling_min_freq", mCpufreq, mCpufreq)); 360 361 if (mSimpleperfMode) { 362 mSpUtil = SimplePerfUtil.newInstance(mTestDevice, SimplePerfType.STAT); 363 if (mSimpleperfArgu.size() == 0) { 364 mSimpleperfArgu.add("-e cpu-cycles:k,cpu-cycles:u"); 365 } 366 mSpUtil.setArgumentList(mSimpleperfArgu); 367 } 368 369 if (mAutoDiscoverCacheInfo) { 370 // Attempt to detect cache path automatically 371 // Expected output look similar to the following: 372 // 373 // > ... vdc dump | grep cache 374 // 0 4123 /dev/block/platform/soc/7824900.sdhci/by-name/cache /cache ext4 rw, \ 375 // seclabel,nosuid,nodev,noatime,discard,data=ordered 0 0 376 if (mTestDevice.enableAdbRoot()) { 377 String output = mTestDevice.executeShellCommand("vdc dump | grep cache"); 378 CLog.d("Output from shell command 'vdc dump | grep cache': %s", output); 379 String[] segments = output.split(" "); 380 if (segments.length >= 3) { 381 mCache = segments[2]; 382 } else { 383 CLog.w("Fail to detect cache path. Fall back to use '%s'", mCache); 384 } 385 } else { 386 CLog.d("Cannot get cache path because device %s is not rooted.", 387 mTestDevice.getSerialNumber()); 388 } 389 390 // Attempt to detect cache partition size automatically 391 // Expected output looks similar to the following: 392 // 393 // > ... df cache 394 // Filesystem 1K-blocks Used Available Use% Mounted on 395 // /dev/block/mmcblk0p34 60400 56 60344 1% /cache 396 String output = mTestDevice.executeShellCommand("df cache"); 397 CLog.d(String.format("Output from shell command 'df cache': %s", output)); 398 String[] lines = output.split("\r?\n"); 399 if (lines.length >= 2) { 400 String[] segments = lines[1].split(" "); 401 if (segments.length >= 2) { 402 if (lines[0].toLowerCase().contains("1k-blocks")) { 403 mCachePartitionSize = Integer.parseInt(segments[1]) / 1024; 404 } else { 405 throw new IllegalArgumentException("Unknown unit for the cache size."); 406 } 407 } 408 } 409 410 CLog.d("cache-device is set to %s ...", mCache); 411 CLog.d("cache-partition-size is set to %d ...", mCachePartitionSize); 412 } 413 } 414 415 /** 416 * Clean up the device by formatting a new cache partition. 417 */ 418 private void cleanUp() throws DeviceNotAvailableException { 419 mTestDevice.executeShellCommand(String.format("make_ext4fs %s", mCache)); 420 } 421 422 /** 423 * {@inheritDoc} 424 */ 425 @Override 426 public void setDevice(ITestDevice device) { 427 mTestDevice = device; 428 } 429 430 /** 431 * {@inheritDoc} 432 */ 433 @Override 434 public ITestDevice getDevice() { 435 return mTestDevice; 436 } 437 } 438 439