1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.performance.tests; 17 18 import com.android.ddmlib.MultiLineReceiver; 19 import com.android.ddmlib.testrunner.TestIdentifier; 20 import com.android.tradefed.config.Option; 21 import com.android.tradefed.config.Option.Importance; 22 import com.android.tradefed.device.DeviceNotAvailableException; 23 import com.android.tradefed.device.ITestDevice; 24 import com.android.tradefed.log.LogUtil.CLog; 25 import com.android.tradefed.result.ITestInvocationListener; 26 import com.android.tradefed.testtype.IDeviceTest; 27 import com.android.tradefed.testtype.IRemoteTest; 28 import com.android.tradefed.util.AbiFormatter; 29 import com.android.tradefed.util.SimplePerfStatResultParser; 30 import com.android.tradefed.util.SimplePerfUtil; 31 import com.android.tradefed.util.SimplePerfUtil.SimplePerfType; 32 import com.android.tradefed.util.SimpleStats; 33 34 import org.junit.Assert; 35 36 import java.text.NumberFormat; 37 import java.text.ParseException; 38 import java.util.ArrayList; 39 import java.util.Collections; 40 import java.util.HashMap; 41 import java.util.List; 42 import java.util.Locale; 43 import java.util.Map; 44 import java.util.regex.Matcher; 45 import java.util.regex.Pattern; 46 47 /** 48 * Test which runs various micro_bench tests and reports the results. 49 * <p> 50 * Tests include: 51 * </p><ul> 52 * <li>{@code micro_bench sleep 1 ITERATIONS}</li> 53 * <li>{@code micro_bench memset ARG ITERATIONS} where ARG is defined by {@code memset-small}</li> 54 * <li>{@code micro_bench memset ARG ITERATIONS} where ARG is defined by {@code memset-large}</li> 55 * <li>{@code micro_bench memcpy ARG ITERATIONS} where ARG is defined by {@code memcpy-small}</li> 56 * <li>{@code micro_bench memcpy ARG ITERATIONS} where ARG is defined by {@code memcpy-large}</li> 57 * <li>{@code micro_bench memread ARG ITERATIONS} where ARG is defined by {@code memread}</li> 58 * </ul> 59 */ 60 public class MicroBenchTest implements IDeviceTest, IRemoteTest { 61 private final static String RUN_KEY = "micro_bench"; 62 63 private final static Pattern SLEEP_PATTERN = Pattern.compile( 64 "sleep\\(\\d+\\) took (\\d+.\\d+) seconds"); 65 private final static Pattern MEMSET_PATTERN = Pattern.compile( 66 "memset \\d+x\\d+ bytes took (\\d+.\\d+) seconds \\((\\d+.\\d+) MB/s\\)"); 67 private final static Pattern MEMCPY_PATTERN = Pattern.compile( 68 "memcpy \\d+x\\d+ bytes took (\\d+.\\d+) seconds \\((\\d+.\\d+) MB/s\\)"); 69 private final static Pattern MEMREAD_PATTERN = Pattern.compile( 70 "memread \\d+x\\d+ bytes took (\\d+.\\d+) seconds \\((\\d+.\\d+) MB/s\\)"); 71 72 private final static String BINARY_NAME = "micro_bench|#ABI32#|"; 73 private final static String SLEEP_CMD = "%s sleep 1 %d"; 74 // memset, memcpy, and memread pass in another argument befor passing the cmd format string to 75 // TestCase, so escape the command and iteration formats. 76 private final static String MEMSET_CMD = "%%s memset %d %%d"; 77 private final static String MEMCPY_CMD = "%%s memcpy %d %%d"; 78 private final static String MEMREAD_CMD = "%%s memread %d %%d"; 79 80 @Option(name = AbiFormatter.FORCE_ABI_STRING, 81 description = AbiFormatter.FORCE_ABI_DESCRIPTION, 82 importance = Importance.IF_UNSET) 83 private String mForceAbi = null; 84 85 @Option(name = "sleep-iterations") 86 private Integer mSleepIterations = 10; 87 88 @Option(name = "memset-iterations") 89 private Integer mMemsetIterations = 100; 90 91 @Option(name = "memset-small") 92 private Integer mMemsetSmall = 8 * 1024; // 8 KB 93 94 @Option(name = "memset-large") 95 private Integer mMemsetLarge = 2 * 1024 * 1024; // 2 MB 96 97 @Option(name = "memcpy-iterations") 98 private Integer mMemcpyIterations = 100; 99 100 @Option(name = "memcpy-small") 101 private Integer mMemcpySmall = 8 * 1024; // 8 KB 102 103 @Option(name = "memcpy-large") 104 private Integer mMemcpyLarge = 2 * 1024 * 1024; // 2 MB 105 106 @Option(name = "memread-iterations") 107 private Integer mMemreadIterations = 100; 108 109 @Option(name = "memread") 110 private Integer mMemreadArg = 8 * 1024; // 8 KB 111 112 @Option(name = "stop-framework", description = "Stop the framework before running tests") 113 private boolean mStopFramework = true; 114 115 @Option(name = "profile-mode", description = 116 "In profile mode, microbench runs memset, memcpy, and " 117 + "memread from 32 bytes to 8 MB, doubling the size each time.") 118 private boolean mProfileMode = false; 119 120 @Option(name = "simpleperf-mode", 121 description = "Whether use simpleperf to get low level metrics") 122 private static boolean mSimpleperfMode = false; 123 124 @Option(name = "simpleperf-argu", description = "simpleperf arguments") 125 private List<String> mSimpleperfArgu = new ArrayList<String>(); 126 127 String mFormattedBinaryName = null; 128 ITestDevice mTestDevice = null; 129 SimplePerfUtil mSpUtil = null; 130 131 /** 132 * Class used to read and parse micro_bench output. 133 */ 134 private static class MicroBenchReceiver extends MultiLineReceiver { 135 private Pattern mPattern; 136 private int mTimeGroup = -1; 137 private int mPerfGroup = -1; 138 private SimpleStats mTimeStats = null; 139 private SimpleStats mPerfStats = null; 140 private Map<String, Double> mSimpleperfMetricsMap = new HashMap<String, Double>(); 141 142 private boolean mSimpleperfOutput; 143 144 /** 145 * Create a receiver for parsing micro_bench output. 146 * 147 * @param pattern The regex {@link Pattern} to parse each line of output. Time and/or 148 * performance numbers should be their own group. 149 * @param timeGroup the index of the time data in the pattern. 150 * @param perfGroup the index of the performance data in the pattern. 151 */ 152 public MicroBenchReceiver(Pattern pattern, int timeGroup, int perfGroup) { 153 mPattern = pattern; 154 mTimeGroup = timeGroup; 155 if (mTimeGroup > 0) { 156 mTimeStats = new SimpleStats(); 157 } 158 mPerfGroup = perfGroup; 159 if (mPerfGroup > 0) { 160 mPerfStats = new SimpleStats(); 161 } 162 mSimpleperfOutput = false; 163 } 164 165 /** 166 * {@inheritDoc} 167 */ 168 @Override 169 public void processNewLines(String[] lines) { 170 for (String line : lines) { 171 if (!mSimpleperfOutput || !mSimpleperfMode) { 172 if (line.contains("Performance counter statistics:")) { 173 mSimpleperfOutput = true; 174 continue; 175 } 176 CLog.v(line); 177 Matcher m = mPattern.matcher(line); 178 if (m.matches()) { 179 if (mTimeStats != null) { 180 mTimeStats.add( 181 Double.valueOf(m.group(mTimeGroup)) * 1000d); // secs to ms 182 } 183 if (mPerfStats != null) { 184 mPerfStats.add(Double.valueOf(m.group(mPerfGroup))); 185 } 186 } 187 } else { 188 List<String> spResult = SimplePerfStatResultParser.parseSingleLine(line); 189 if (spResult.size() == 1) { 190 CLog.d(String.format("Total run time: %s", spResult.get(0))); 191 } else if (spResult.size() == 3) { 192 try { 193 Double metricValue = NumberFormat.getNumberInstance(Locale.US) 194 .parse(spResult.get(0)).doubleValue(); 195 mSimpleperfMetricsMap.put(spResult.get(1), metricValue); 196 } catch (ParseException e) { 197 CLog.e("Simpleperf metrics parse failure: " + e.toString()); 198 } 199 } else { 200 CLog.d("Skipping line: " + line); 201 } 202 } 203 } 204 } 205 206 /** 207 * Get the time stats. 208 * 209 * @return a {@link SimpleStats} object containing the time stats in ms. 210 */ 211 public SimpleStats getTimeStats() { 212 return mTimeStats; 213 } 214 215 /** 216 * Get the performance stats. 217 * 218 * @return a {@link SimpleStats} object containing the performance stats in MB/s. 219 */ 220 public SimpleStats getPerfStats() { 221 return mPerfStats; 222 } 223 224 public Map<String, Double> getSimpleperfMetricsMap() { 225 return mSimpleperfMetricsMap; 226 } 227 228 /** 229 * {@inheritDoc} 230 */ 231 @Override 232 public boolean isCancelled() { 233 return false; 234 } 235 } 236 237 /** 238 * Class used to hold test data and run individual tests. 239 */ 240 private class TestCase { 241 private String mTestName; 242 private String mTestKey; 243 private String mCommand; 244 private int mIterations = 0; 245 private MicroBenchReceiver mReceiver; 246 247 /** 248 * Create a test case. 249 * 250 * @param testName The name of the test 251 * @param testKey The key of the test used to report metrics. 252 * @param command The command to run. Must contain a {@code %d} in the place of the number 253 * of iterations. 254 * @param iterations The amount of iterations to run. 255 * @param pattern The regex {@link Pattern} to parse each line of output. Time and/or 256 * performance numbers should be their own group. 257 * @param timeGroup the index of the time data in the pattern. 258 * @param perfGroup the index of the performance data in the pattern. 259 */ 260 public TestCase(String testName, String testKey, String command, int iterations, 261 Pattern pattern, int timeGroup, int perfGroup) { 262 mTestName = testName; 263 mTestKey = testKey; 264 mCommand = command; 265 mIterations = iterations; 266 mReceiver = new MicroBenchReceiver(pattern, timeGroup, perfGroup); 267 } 268 269 /** 270 * Runs the test and adds the results to the test metrics. 271 * 272 * @param listener the {@link ITestInvocationListener}. 273 * @param metrics the metrics map for the results. 274 * @throws DeviceNotAvailableException if the device is not available while running the 275 * test. 276 */ 277 public void runTest(ITestInvocationListener listener, Map<String, String> metrics) 278 throws DeviceNotAvailableException { 279 if (mIterations == 0) { 280 return; 281 } 282 283 CLog.i("Running %s test", mTestName); 284 285 TestIdentifier testId = new TestIdentifier(getClass().getCanonicalName(), 286 mTestKey); 287 listener.testStarted(testId); 288 final String cmd = String.format(mCommand, mFormattedBinaryName, mIterations); 289 290 if (mSimpleperfMode) { 291 mSpUtil.executeCommand(cmd, mReceiver); 292 } else { 293 mTestDevice.executeShellCommand(cmd, mReceiver); 294 } 295 296 Boolean timePass = parseStats(mReceiver.getTimeStats(), "time", metrics); 297 Boolean perfPass = parseStats(mReceiver.getPerfStats(), "perf", metrics); 298 299 for (Map.Entry<String, Double> entry : 300 mReceiver.getSimpleperfMetricsMap().entrySet()) { 301 metrics.put(String.format("%s_%s", mTestKey, entry.getKey()), 302 Double.toString(entry.getValue() / mIterations)); 303 } 304 305 if (!timePass || !perfPass) { 306 listener.testFailed(testId, 307 "Iteration count mismatch (see host log)."); 308 } 309 Map<String, String> emptyMap = Collections.emptyMap(); 310 listener.testEnded(testId, emptyMap); 311 } 312 313 /** 314 * Parse the stats from {@link MicroBenchReceiver} and add them to the metrics 315 * 316 * @param stats {@link SimpleStats} from {@link MicroBenchReceiver} or {@code null} if 317 * there are no stats. 318 * @param category the category of the stats, either {@code perf} or {@code time}. 319 * @param metrics the metrics map to add the parsed stats to. 320 * @return {@code true} if the stats were successfully parsed and added or if 321 * {@code stats == null}, {@code false} if there was an error. 322 */ 323 private boolean parseStats(SimpleStats stats, String category, 324 Map<String, String> metrics) { 325 if (stats == null) { 326 return true; 327 } 328 329 String unit = "perf".equals(category) ? "MB/s" : "ms"; 330 331 CLog.i("%s %s results: size = %d, mean = %f %s, stdev = %f %s", mTestName, category, 332 stats.size(), stats.mean(), unit, stats.stdev(), unit); 333 334 if (stats.mean() == null) { 335 CLog.e("Expected non null mean for %s %s", mTestName, category); 336 return false; 337 } 338 339 if (stats.size() != mIterations) { 340 CLog.e("Expected iterations (%d) not equal to sample size (%d) for %s %s", 341 mIterations, stats.size(), mTestName, category); 342 return false; 343 } 344 345 if (metrics != null) { 346 metrics.put(String.format("%s_%s", mTestKey, category), 347 Double.toString(stats.mean())); 348 } 349 return true; 350 } 351 } 352 353 /** 354 * {@inheritDoc} 355 */ 356 @Override 357 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 358 String name, cmd; 359 List<TestCase> testCases = new ArrayList<TestCase>(); 360 361 mFormattedBinaryName = AbiFormatter.formatCmdForAbi(BINARY_NAME, mForceAbi); 362 363 Assert.assertNotNull(mTestDevice); 364 // TODO: Replace this with a more generic way to check for the presence of a binary 365 final String output = mTestDevice.executeShellCommand(mFormattedBinaryName); 366 Assert.assertFalse(output.contains(mFormattedBinaryName + ": not found")); 367 368 if (mSimpleperfMode) { 369 mSpUtil = SimplePerfUtil.newInstance(mTestDevice, SimplePerfType.STAT); 370 if (mSimpleperfArgu.size() == 0) { 371 mSimpleperfArgu.add("-e cpu-cycles:k,cpu-cycles:u"); 372 } 373 mSpUtil.setArgumentList(mSimpleperfArgu); 374 } 375 376 if (mStopFramework) { 377 mTestDevice.executeShellCommand("stop"); 378 } 379 380 if (mProfileMode) { 381 for (int i = 1 << 5; i < 1 << 24; i <<= 1) { 382 name = String.format("memset %d", i); 383 cmd = String.format(MEMSET_CMD, i); 384 testCases.add(new TestCase(name, String.format("memset_%d", i), cmd, 385 mMemsetIterations, MEMSET_PATTERN, 1, 2)); 386 } 387 388 for (int i = 1 << 5; i < 1 << 24; i <<= 1) { 389 name = String.format("memcpy %d", i); 390 cmd = String.format(MEMCPY_CMD, i); 391 testCases.add(new TestCase(name, String.format("memcpy_%d", i), cmd, 392 mMemcpyIterations, MEMCPY_PATTERN, 1, 2)); 393 } 394 395 for (int i = 1 << 5; i < 1 << 24; i <<= 1) { 396 name = String.format("memread %d", i); 397 cmd = String.format(MEMREAD_CMD, i); 398 testCases.add(new TestCase(name, String.format("memread_%d", i), cmd, 399 mMemreadIterations, MEMREAD_PATTERN, 1, 2)); 400 } 401 402 // Don't want to report test metrics in profile mode 403 for (TestCase test : testCases) { 404 test.runTest(listener, null); 405 } 406 } else { 407 // Sleep test case 408 testCases.add(new TestCase("sleep", "sleep", SLEEP_CMD, mSleepIterations, SLEEP_PATTERN, 409 1, -1)); 410 411 // memset small test case 412 name = String.format("memset %d", mMemsetSmall); 413 cmd = String.format(MEMSET_CMD, mMemsetSmall); 414 testCases.add(new TestCase(name, "memset_small", cmd, mMemsetIterations, MEMSET_PATTERN, 415 1, 2)); 416 417 // memset large test case 418 name = String.format("memset %d", mMemsetLarge); 419 cmd = String.format(MEMSET_CMD, mMemsetLarge); 420 testCases.add(new TestCase(name, "memset_large", cmd, mMemsetIterations, MEMSET_PATTERN, 421 1, 2)); 422 423 // memcpy small test case 424 name = String.format("memcpy %d", mMemcpySmall); 425 cmd = String.format(MEMCPY_CMD, mMemcpySmall); 426 testCases.add(new TestCase(name, "memcpy_small", cmd, mMemcpyIterations, MEMCPY_PATTERN, 427 1, 2)); 428 429 // memcpy large test case 430 name = String.format("memcpy %d", mMemcpyLarge); 431 cmd = String.format(MEMCPY_CMD, mMemcpyLarge); 432 testCases.add(new TestCase(name, "memcpy_large", cmd, mMemcpyIterations, MEMCPY_PATTERN, 433 1, 2)); 434 435 // memread test case 436 name = String.format("memread %d", mMemreadArg); 437 cmd = String.format(MEMREAD_CMD, mMemreadArg); 438 testCases.add(new TestCase(name, "memread", cmd, mMemreadIterations, MEMREAD_PATTERN, 439 1, 2)); 440 441 listener.testRunStarted(RUN_KEY, testCases.size()); 442 long beginTime = System.currentTimeMillis(); 443 Map<String, String> metrics = new HashMap<String, String>(); 444 for (TestCase test : testCases) { 445 test.runTest(listener, metrics); 446 } 447 listener.testRunEnded((System.currentTimeMillis() - beginTime), metrics); 448 } 449 450 if (mStopFramework) { 451 mTestDevice.executeShellCommand("start"); 452 } 453 } 454 455 /** 456 * {@inheritDoc} 457 */ 458 @Override 459 public void setDevice(ITestDevice device) { 460 mTestDevice = device; 461 } 462 463 /** 464 * {@inheritDoc} 465 */ 466 @Override 467 public ITestDevice getDevice() { 468 return mTestDevice; 469 } 470 } 471