1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.performance.tests; 18 19 import com.android.ddmlib.IDevice; 20 import com.android.ddmlib.MultiLineReceiver; 21 import com.android.ddmlib.NullOutputReceiver; 22 import com.android.tradefed.config.Option; 23 import com.android.tradefed.device.DeviceNotAvailableException; 24 import com.android.tradefed.device.ITestDevice; 25 import com.android.tradefed.log.LogUtil.CLog; 26 import com.android.tradefed.result.FileInputStreamSource; 27 import com.android.tradefed.result.ITestInvocationListener; 28 import com.android.tradefed.result.InputStreamSource; 29 import com.android.tradefed.result.LogDataType; 30 import com.android.tradefed.testtype.IDeviceTest; 31 import com.android.tradefed.testtype.IRemoteTest; 32 import com.android.tradefed.util.FileUtil; 33 import com.android.tradefed.util.StreamUtil; 34 35 import junit.framework.TestCase; 36 37 import org.junit.Assert; 38 39 import java.io.File; 40 import java.util.HashMap; 41 import java.util.HashSet; 42 import java.util.LinkedList; 43 import java.util.List; 44 import java.util.Map; 45 import java.util.Map.Entry; 46 import java.util.Set; 47 import java.util.concurrent.TimeUnit; 48 49 /** 50 * Runs the FIO benchmarks. 51 * <p> 52 * This test pushes the FIO executable to the device and runs several benchmarks. Running each 53 * benchmark consists of creating a config file, creating one or more data files, clearing the disk 54 * cache and then running FIO. The test runs a variety of different configurations including a 55 * simple benchmark with a single thread, a storage benchmark with 4 threads, a media server 56 * emulator, and a media scanner emulator. 57 * </p> 58 */ 59 public class FioBenchmarkTest implements IDeviceTest, IRemoteTest { 60 // TODO: Refactor this to only pick out fields we care about. 61 private static final String[] FIO_V0_RESULT_FIELDS = { 62 "jobname", "groupid", "error", 63 // Read stats 64 "read-kb-io", "read-bandwidth", "read-runtime", 65 "read-slat-min", "read-slat-max", "read-slat-mean", "read-slat-stddev", 66 "read-clat-min", "read-clat-max", "read-clat-mean", "read-clat-stddev", 67 "read-bandwidth-min", "read-bandwidth-max", "read-bandwidth-percent", "read-bandwidth-mean", 68 "read-bandwidth-stddev", 69 // Write stats 70 "write-kb-io", "write-bandwidth", "write-runtime", 71 "write-slat-min", "write-slat-max", "write-slat-mean", "write-slat-stddev", 72 "write-clat-min", "write-clat-max", "write-clat-mean", "write-clat-stddev", 73 "write-bandwidth-min", "write-bandwidth-max", "write-bandwidth-percent", 74 "write-bandwidth-mean", "write-bandwidth-stddev", 75 // CPU stats 76 "cpu-user", "cpu-system", "cpu-context-switches", "cpu-major-page-faults", 77 "cpu-minor-page-faults", 78 // IO depth stats 79 "io-depth-1", "io-depth-2", "io-depth-4", "io-depth-8", "io-depth-16", "io-depth-32", 80 "io-depth-64", 81 // IO lat stats 82 "io-lat-2-ms", "io-lat-4-ms", "io-lat-10-ms", "io-lat-20-ms", "io-lat-50-ms", 83 "io-lat-100-ms", "io-lat-250-ms", "io-lat-500-ms", "io-lat-750-ms", "io-lat-1000-ms", 84 "io-lat-2000-ms" 85 }; 86 private static final String[] FIO_V3_RESULT_FIELDS = { 87 "terse-version", "fio-version", "jobname", "groupid", "error", 88 // Read stats 89 "read-kb-io", "read-bandwidth", "read-iops", "read-runtime", 90 "read-slat-min", "read-slat-max", "read-slat-mean", "read-slat-stddev", 91 "read-clat-min", "read-clat-max", "read-clat-mean", "read-clat-stddev", 92 null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, 93 null, null, null, null, null, 94 "read-lat-min", "read-lat-max", "read-lat-mean", "read-lat-stddev", 95 "read-bandwidth-min", "read-bandwidth-max", "read-bandwidth-percent", "read-bandwidth-mean", 96 "read-bandwidth-stddev", 97 // Write stats 98 "write-kb-io", "write-bandwidth", "write-iops", "write-runtime", 99 "write-slat-min", "write-slat-max", "write-slat-mean", "write-slat-stddev", 100 "write-clat-min", "write-clat-max", "write-clat-mean", "write-clat-stddev", 101 null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, 102 null, null, null, null, null, 103 "write-lat-min", "write-lat-max", "write-lat-mean", "write-lat-stddev", 104 "write-bandwidth-min", "write-bandwidth-max", "write-bandwidth-percent", 105 "write-bandwidth-mean", "write-bandwidth-stddev", 106 // CPU stats 107 "cpu-user", "cpu-system", "cpu-context-switches", "cpu-major-page-faults", 108 "cpu-minor-page-faults", 109 // IO depth stats 110 "io-depth-1", "io-depth-2", "io-depth-4", "io-depth-8", "io-depth-16", "io-depth-32", 111 "io-depth-64", 112 // IO lat stats 113 "io-lat-2-us", "io-lat-4-us", "io-lat-10-us", "io-lat-20-us", "io-lat-50-us", 114 "io-lat-100-us", "io-lat-250-us", "io-lat-500-us", "io-lat-750-us", "io-lat-1000-us", 115 "io-lat-2-ms", "io-lat-4-ms", "io-lat-10-ms", "io-lat-20-ms", "io-lat-50-ms", 116 "io-lat-100-ms", "io-lat-250-ms", "io-lat-500-ms", "io-lat-750-ms", "io-lat-1000-ms", 117 "io-lat-2000-ms", "io-lat-greater" 118 }; 119 120 private List<TestInfo> mTestCases = null; 121 122 /** 123 * Holds info about a job. The job translates into a job in the FIO config file. Contains the 124 * job name and a map from keys to values. 125 */ 126 private static class JobInfo { 127 public String mJobName = null; 128 public Map<String, String> mParameters = new HashMap<>(); 129 130 /** 131 * Gets the job as a string. 132 * 133 * @return a string of the job formatted for the config file. 134 */ 135 public String createJob() { 136 if (mJobName == null) { 137 return ""; 138 } 139 StringBuilder sb = new StringBuilder(); 140 sb.append(String.format("[%s]\n", mJobName)); 141 for (Entry<String, String> parameter : mParameters.entrySet()) { 142 if (parameter.getValue() == null) { 143 sb.append(String.format("%s\n", parameter.getKey())); 144 } else { 145 sb.append(String.format("%s=%s\n", parameter.getKey(), parameter.getValue())); 146 } 147 } 148 return sb.toString(); 149 } 150 } 151 152 /** 153 * Holds info about a file used in the benchmark. Because of limitations in FIO on Android, 154 * the file needs to be created before the tests are run. Contains the file name and size in kB. 155 */ 156 private static class TestFileInfo { 157 public String mFileName = null; 158 public int mSize = -1; 159 } 160 161 /** 162 * Holds info about the perf metric that are cared about for a given job. 163 */ 164 private static class PerfMetricInfo { 165 public enum ResultType { 166 STRING, INT, FLOAT, PERCENT; 167 168 String value(String input) { 169 switch(this) { 170 case STRING: 171 case INT: 172 case FLOAT: 173 return input; 174 case PERCENT: 175 if (input.length() < 2 || !input.endsWith("%")) { 176 return null; 177 } 178 try { 179 return String.format("%f", Double.parseDouble( 180 input.substring(0, input.length() - 1)) / 100); 181 } catch (NumberFormatException e) { 182 return null; 183 } 184 default: return null; 185 } 186 } 187 } 188 189 public String mJobName = null; 190 public String mFieldName = null; 191 public String mPostKey = null; 192 public ResultType mType = ResultType.STRING; 193 } 194 195 /** 196 * Holds the info associated with a test. 197 * <p> 198 * Contains the test name, key, a list of {@link JobInfo}, a set of {@link TestFileInfo}, 199 * and a set of {@link PerfMetricInfo}. 200 * </p> 201 */ 202 private static class TestInfo { 203 public String mTestName = null; 204 public String mKey = null; 205 public List<JobInfo> mJobs = new LinkedList<>(); 206 public Set<TestFileInfo> mTestFiles = new HashSet<>(); 207 public Set<PerfMetricInfo> mPerfMetrics = new HashSet<>(); 208 209 /** 210 * Gets the config file. 211 * 212 * @return a string containing the contents of the config file needed to run the benchmark. 213 */ 214 private String createConfig() { 215 StringBuilder sb = new StringBuilder(); 216 for (JobInfo job : mJobs) { 217 sb.append(String.format("%s\n", job.createJob())); 218 } 219 return sb.toString(); 220 } 221 } 222 223 /** 224 * Parses the output of the FIO and allows the values to be looked up by job name and property. 225 */ 226 private static class FioParser extends MultiLineReceiver { 227 public Map<String, Map<String, String>> mResults = new HashMap<>(); 228 229 /** 230 * Gets the result for a job and property, or null if the job or the property do not exist. 231 * 232 * @param job the name of the job. 233 * @param property the name of the property. See {@code FIO_RESULT_FIELDS}. 234 * @return the fio results for the job and property or null if it does not exist. 235 */ 236 public String getResult(String job, String property) { 237 if (!mResults.containsKey(job)) { 238 return null; 239 } 240 return mResults.get(job).get(property); 241 } 242 243 /** 244 * {@inheritDoc} 245 */ 246 @Override 247 public void processNewLines(String[] lines) { 248 for (String line : lines) { 249 CLog.d(line); 250 String[] fields = line.split(";"); 251 if (fields.length < FIO_V0_RESULT_FIELDS.length) { 252 continue; 253 } 254 if (fields.length < FIO_V3_RESULT_FIELDS.length) { 255 Map<String, String> r = new HashMap<>(); 256 for (int i = 0; i < FIO_V0_RESULT_FIELDS.length; i++) { 257 r.put(FIO_V0_RESULT_FIELDS[i], fields[i]); 258 } 259 mResults.put(fields[0], r); // Job name is index 0 260 } else if ("3".equals(fields[0])) { 261 Map<String, String> r = new HashMap<>(); 262 for (int i = 0; i < FIO_V3_RESULT_FIELDS.length; i++) { 263 r.put(FIO_V3_RESULT_FIELDS[i], fields[i]); 264 } 265 mResults.put(fields[2], r); // Job name is index 2 266 } else { 267 Assert.fail("Unknown fio terse output version"); 268 } 269 } 270 } 271 272 /** 273 * {@inheritDoc} 274 */ 275 @Override 276 public boolean isCancelled() { 277 return false; 278 } 279 } 280 281 ITestDevice mTestDevice = null; 282 283 private String mFioDir = null; 284 private String mFioBin = null; 285 private String mFioConfig = null; 286 287 @Option(name="fio-location", description="The path to the precompiled FIO executable. If " 288 + "unset, try to use fio from the system image.") 289 private String mFioLocation = null; 290 291 @Option(name="tmp-dir", description="The directory used for interal benchmarks.") 292 private String mTmpDir = "/data/tmp/fio"; 293 294 @Option(name="internal-test-dir", description="The directory used for interal benchmarks.") 295 private String mInternalTestDir = "/data/fio/data"; 296 297 @Option(name="media-test-dir", description="The directory used for media benchmarks.") 298 private String mMediaTestDir = "${EXTERNAL_STORAGE}/fio"; 299 300 @Option(name="external-test-dir", description="The directory used for external benchmarks.") 301 private String mExternalTestDir = "${EXTERNAL_STORAGE}/fio"; 302 303 @Option(name="collect-yaffs-logs", description="Collect yaffs logs before and after tests") 304 private Boolean mCollectYaffsLogs = false; 305 306 @Option(name="run-simple-internal-test", description="Run the simple internal benchmark.") 307 private Boolean mRunSimpleInternalTest = true; 308 309 @Option(name="simple-internal-file-size", 310 description="The file size of the simple internal benchmark in MB.") 311 private int mSimpleInternalFileSize = 256; 312 313 @Option(name="run-simple-external-test", description="Run the simple external benchmark.") 314 private Boolean mRunSimpleExternalTest = false; 315 316 @Option(name="simple-external-file-size", 317 description="The file size of the simple external benchmark in MB.") 318 private int mSimpleExternalFileSize = 256; 319 320 @Option(name="run-storage-internal-test", description="Run the storage internal benchmark.") 321 private Boolean mRunStorageInternalTest = true; 322 323 @Option(name="storage-internal-file-size", 324 description="The file size of the storage internal benchmark in MB.") 325 private int mStorageInternalFileSize = 256; 326 327 @Option(name="storage-internal-job-count", 328 description="The number of jobs for the storage internal benchmark.") 329 private int mStorageInternalJobCount = 4; 330 331 @Option(name="run-storage-external-test", description="Run the storage external benchmark.") 332 private Boolean mRunStorageExternalTest = false; 333 334 @Option(name="storage-external-file-size", 335 description="The file size of the storage external benchmark in MB.") 336 private int mStorageExternalFileSize = 256; 337 338 @Option(name="storage-external-job-count", 339 description="The number of jobs for the storage external benchmark.") 340 private int mStorageExternalJobCount = 4; 341 342 @Option(name="run-media-server-test", description="Run the media server benchmark.") 343 private Boolean mRunMediaServerTest = false; 344 345 @Option(name="media-server-duration", 346 description="The duration of the media server benchmark in secs.") 347 private long mMediaServerDuration = 30; 348 349 @Option(name="media-server-media-file-size", 350 description="The media file size of the media server benchmark in MB.") 351 private int mMediaServerMediaFileSize = 256; 352 353 @Option(name="media-server-worker-file-size", 354 description="The worker file size of the media server benchmark in MB.") 355 private int mMediaServerWorkerFileSize = 256; 356 357 @Option(name="media-server-worker-job-count", 358 description="The number of worker jobs for the media server benchmark.") 359 private int mMediaServerWorkerJobCount = 4; 360 361 @Option(name="run-media-scanner-test", description="Run the media scanner benchmark.") 362 private Boolean mRunMediaScannerTest = false; 363 364 @Option(name="media-scanner-media-file-size", 365 description="The media file size of the media scanner benchmark in kB.") 366 private int mMediaScannerMediaFileSize = 8; 367 368 @Option(name="media-scanner-media-file-count", 369 description="The number of media files to scan.") 370 private int mMediaScannerMediaFileCount = 256; 371 372 @Option(name="media-scanner-worker-file-size", 373 description="The worker file size of the media scanner benchmark in MB.") 374 private int mMediaScannerWorkerFileSize = 256; 375 376 @Option(name="media-scanner-worker-job-count", 377 description="The number of worker jobs for the media server benchmark.") 378 private int mMediaScannerWorkerJobCount = 4; 379 380 @Option(name="key-suffix", 381 description="The suffix to add to the reporting key in order to override the default") 382 private String mKeySuffix = null; 383 384 /** 385 * Sets up all the benchmarks. 386 */ 387 private void setupTests() { 388 if (mTestCases != null) { 389 // assume already set up 390 return; 391 } 392 393 mTestCases = new LinkedList<>(); 394 395 if (mRunSimpleInternalTest) { 396 addSimpleTest("read", "sync", true); 397 addSimpleTest("write", "sync", true); 398 addSimpleTest("randread", "sync", true); 399 addSimpleTest("randwrite", "sync", true); 400 addSimpleTest("randread", "mmap", true); 401 addSimpleTest("randwrite", "mmap", true); 402 } 403 404 if (mRunSimpleExternalTest) { 405 addSimpleTest("read", "sync", false); 406 addSimpleTest("write", "sync", false); 407 addSimpleTest("randread", "sync", false); 408 addSimpleTest("randwrite", "sync", false); 409 addSimpleTest("randread", "mmap", false); 410 addSimpleTest("randwrite", "mmap", false); 411 } 412 413 if (mRunStorageInternalTest) { 414 addStorageTest("read", "sync", true); 415 addStorageTest("write", "sync", true); 416 addStorageTest("randread", "sync", true); 417 addStorageTest("randwrite", "sync", true); 418 addStorageTest("randread", "mmap", true); 419 addStorageTest("randwrite", "mmap", true); 420 } 421 422 if (mRunStorageExternalTest) { 423 addStorageTest("read", "sync", false); 424 addStorageTest("write", "sync", false); 425 addStorageTest("randread", "sync", false); 426 addStorageTest("randwrite", "sync", false); 427 addStorageTest("randread", "mmap", false); 428 addStorageTest("randwrite", "mmap", false); 429 } 430 431 if (mRunMediaServerTest) { 432 addMediaServerTest("read"); 433 addMediaServerTest("write"); 434 } 435 436 if (mRunMediaScannerTest) { 437 addMediaScannerTest(); 438 } 439 } 440 441 /** 442 * Sets up the simple FIO benchmark. 443 * <p> 444 * The test consists of a single process reading or writing to a file. 445 * </p> 446 * 447 * @param rw the type of IO pattern. One of {@code read}, {@code write}, {@code randread}, or 448 * {@code randwrite}. 449 * @param ioengine defines how the job issues I/O. Such as {@code sync}, {@code vsync}, 450 * {@code mmap}, or {@code cpuio} and others. 451 * @param internal whether the test should be run on the internal (/data) partition or the 452 * external partition. 453 */ 454 private void addSimpleTest(String rw, String ioengine, boolean internal) { 455 String fileName = "testfile"; 456 String jobName = "job"; 457 458 String directory; 459 int fileSize; 460 461 TestInfo t = new TestInfo(); 462 if (internal) { 463 t.mTestName = String.format("SimpleBenchmark-int-%s-%s", ioengine, rw); 464 t.mKey = "fio_simple_int_benchmark"; 465 directory = mInternalTestDir; 466 fileSize = mSimpleInternalFileSize; 467 } else { 468 t.mTestName = String.format("SimpleBenchmark-ext-%s-%s", ioengine, rw); 469 t.mKey = "fio_simple_ext_benchmark"; 470 directory = mExternalTestDir; 471 fileSize = mSimpleExternalFileSize; 472 } 473 474 TestFileInfo f = new TestFileInfo(); 475 f.mFileName = new File(directory, fileName).getAbsolutePath(); 476 f.mSize = fileSize * 1024; // fileSize is in MB but we want it in kB. 477 t.mTestFiles.add(f); 478 479 JobInfo j = new JobInfo(); 480 j.mJobName = jobName; 481 j.mParameters.put("directory", directory); 482 j.mParameters.put("filename", fileName); 483 j.mParameters.put("fsync", "1024"); 484 j.mParameters.put("ioengine", ioengine); 485 j.mParameters.put("rw", rw); 486 j.mParameters.put("size", String.format("%dM",fileSize)); 487 t.mJobs.add(j); 488 489 PerfMetricInfo m = new PerfMetricInfo(); 490 m.mJobName = jobName; 491 if ("sync".equals(ioengine)) { 492 m.mPostKey = String.format("%s_bandwidth", rw); 493 } else { 494 m.mPostKey = String.format("%s_%s_bandwidth", rw, ioengine); 495 } 496 m.mType = PerfMetricInfo.ResultType.FLOAT; 497 if (rw.endsWith("read")) { 498 m.mFieldName = "read-bandwidth-mean"; 499 } else if (rw.endsWith("write")) { 500 m.mFieldName = "write-bandwidth-mean"; 501 } 502 t.mPerfMetrics.add(m); 503 504 mTestCases.add(t); 505 } 506 507 /** 508 * Sets up the storage FIO benchmark. 509 * <p> 510 * The test consists of several processes reading or writing to a file. 511 * </p> 512 * 513 * @param rw the type of IO pattern. One of {@code read}, {@code write}, {@code randread}, or 514 * {@code randwrite}. 515 * @param ioengine defines how the job issues I/O. Such as {@code sync}, {@code vsync}, 516 * {@code mmap}, or {@code cpuio} and others. 517 * @param internal whether the test should be run on the internal (/data) partition or the 518 * external partition. 519 */ 520 private void addStorageTest(String rw, String ioengine, boolean internal) { 521 String fileName = "testfile"; 522 String jobName = "workers"; 523 524 String directory; 525 int fileSize; 526 int jobCount; 527 528 TestInfo t = new TestInfo(); 529 if (internal) { 530 t.mTestName = String.format("StorageBenchmark-int-%s-%s", ioengine, rw); 531 t.mKey = "fio_storage_int_benchmark"; 532 directory = mInternalTestDir; 533 fileSize = mStorageInternalFileSize; 534 jobCount = mStorageInternalJobCount; 535 } else { 536 t.mTestName = String.format("StorageBenchmark-ext-%s-%s", ioengine, rw); 537 t.mKey = "fio_storage_ext_benchmark"; 538 directory = mExternalTestDir; 539 fileSize = mStorageExternalFileSize; 540 jobCount = mStorageExternalJobCount; 541 } 542 543 TestFileInfo f = new TestFileInfo(); 544 f.mFileName = new File(directory, fileName).getAbsolutePath(); 545 f.mSize = fileSize * 1024; // fileSize is in MB but we want it in kB. 546 t.mTestFiles.add(f); 547 548 JobInfo j = new JobInfo(); 549 j.mJobName = jobName; 550 j.mParameters.put("directory", directory); 551 j.mParameters.put("filename", fileName); 552 j.mParameters.put("fsync", "1024"); 553 j.mParameters.put("group_reporting", null); 554 j.mParameters.put("ioengine", ioengine); 555 j.mParameters.put("new_group", null); 556 j.mParameters.put("numjobs", String.format("%d", jobCount)); 557 j.mParameters.put("rw", rw); 558 j.mParameters.put("size", String.format("%dM",fileSize)); 559 t.mJobs.add(j); 560 561 PerfMetricInfo m = new PerfMetricInfo(); 562 m.mJobName = jobName; 563 if ("sync".equals(ioengine)) { 564 m.mPostKey = String.format("%s_bandwidth", rw); 565 } else { 566 m.mPostKey = String.format("%s_%s_bandwidth", rw, ioengine); 567 } 568 m.mType = PerfMetricInfo.ResultType.FLOAT; 569 if (rw.endsWith("read")) { 570 m.mFieldName = "read-bandwidth-mean"; 571 } else if (rw.endsWith("write")) { 572 m.mFieldName = "write-bandwidth-mean"; 573 } 574 t.mPerfMetrics.add(m); 575 576 m = new PerfMetricInfo(); 577 m.mJobName = jobName; 578 if ("sync".equals(ioengine)) { 579 m.mPostKey = String.format("%s_latency", rw); 580 } else { 581 m.mPostKey = String.format("%s_%s_latency", rw, ioengine); 582 } 583 m.mType = PerfMetricInfo.ResultType.FLOAT; 584 if (rw.endsWith("read")) { 585 m.mFieldName = "read-clat-mean"; 586 } else if (rw.endsWith("write")) { 587 m.mFieldName = "write-clat-mean"; 588 } 589 t.mPerfMetrics.add(m); 590 591 mTestCases.add(t); 592 } 593 594 /** 595 * Sets up the media server benchmark. 596 * <p> 597 * The test consists of a single process at a higher priority reading or writing to a file 598 * while several other worker processes read and write to a different file. 599 * </p> 600 * 601 * @param rw the type of IO pattern. One of {@code read}, {@code write} 602 */ 603 private void addMediaServerTest(String rw) { 604 String mediaJob = "media-server"; 605 String mediaFile = "mediafile"; 606 String workerJob = "workers"; 607 String workerFile = "workerfile"; 608 609 TestInfo t = new TestInfo(); 610 t.mTestName = String.format("MediaServerBenchmark-%s", rw); 611 t.mKey = "fio_media_server_benchmark"; 612 613 TestFileInfo f = new TestFileInfo(); 614 f.mFileName = new File(mMediaTestDir, mediaFile).getAbsolutePath(); 615 f.mSize = mMediaServerMediaFileSize * 1024; // File size is in MB but we want it in kB. 616 t.mTestFiles.add(f); 617 618 f = new TestFileInfo(); 619 f.mFileName = new File(mMediaTestDir, workerFile).getAbsolutePath(); 620 f.mSize = mMediaServerWorkerFileSize * 1024; // File size is in MB but we want it in kB. 621 t.mTestFiles.add(f); 622 623 JobInfo j = new JobInfo(); 624 j.mJobName = "global"; 625 j.mParameters.put("directory", mMediaTestDir); 626 j.mParameters.put("fsync", "1024"); 627 j.mParameters.put("ioengine", "sync"); 628 j.mParameters.put("runtime", String.format("%d", mMediaServerDuration)); 629 j.mParameters.put("time_based", null); 630 t.mJobs.add(j); 631 632 j = new JobInfo(); 633 j.mJobName = mediaJob; 634 j.mParameters.put("filename", mediaFile); 635 j.mParameters.put("iodepth", "32"); 636 j.mParameters.put("nice", "-16"); 637 j.mParameters.put("rate", "6m"); 638 j.mParameters.put("rw", rw); 639 j.mParameters.put("size", String.format("%dM", mMediaServerMediaFileSize)); 640 t.mJobs.add(j); 641 642 j = new JobInfo(); 643 j.mJobName = workerJob; 644 j.mParameters.put("filename", workerFile); 645 j.mParameters.put("group_reporting", null); 646 j.mParameters.put("new_group", null); 647 j.mParameters.put("nice", "0"); 648 j.mParameters.put("numjobs", String.format("%d", mMediaServerWorkerJobCount)); 649 j.mParameters.put("rw", "randrw"); 650 j.mParameters.put("size", String.format("%dM", mMediaServerWorkerFileSize)); 651 t.mJobs.add(j); 652 653 PerfMetricInfo m = new PerfMetricInfo(); 654 m.mJobName = mediaJob; 655 m.mPostKey = String.format("%s_media_bandwidth", rw); 656 m.mType = PerfMetricInfo.ResultType.FLOAT; 657 if (rw.endsWith("read")) { 658 m.mFieldName = "read-bandwidth-mean"; 659 } else if (rw.endsWith("write")) { 660 m.mFieldName = "write-bandwidth-mean"; 661 } 662 t.mPerfMetrics.add(m); 663 664 m = new PerfMetricInfo(); 665 m.mJobName = mediaJob; 666 m.mPostKey = String.format("%s_media_latency", rw); 667 m.mType = PerfMetricInfo.ResultType.FLOAT; 668 if (rw.endsWith("read")) { 669 m.mFieldName = "read-clat-mean"; 670 } else if (rw.endsWith("write")) { 671 m.mFieldName = "write-clat-mean"; 672 } 673 t.mPerfMetrics.add(m); 674 675 m = new PerfMetricInfo(); 676 m.mJobName = workerJob; 677 m.mPostKey = String.format("%s_workers_read_bandwidth", rw); 678 m.mFieldName = "read-bandwidth-mean"; 679 m.mType = PerfMetricInfo.ResultType.FLOAT; 680 t.mPerfMetrics.add(m); 681 682 m = new PerfMetricInfo(); 683 m.mJobName = workerJob; 684 m.mPostKey = String.format("%s_workers_write_bandwidth", rw); 685 m.mFieldName = "write-bandwidth-mean"; 686 m.mType = PerfMetricInfo.ResultType.FLOAT; 687 t.mPerfMetrics.add(m); 688 689 mTestCases.add(t); 690 } 691 692 /** 693 * Sets up the media scanner benchmark. 694 * <p> 695 * The test consists of a single process reading several small files while several other worker 696 * processes read and write to a different file. 697 * </p> 698 */ 699 private void addMediaScannerTest() { 700 String mediaJob = "media-server"; 701 String mediaFile = "mediafile.%d"; 702 String workerJob = "workers"; 703 String workerFile = "workerfile"; 704 705 TestInfo t = new TestInfo(); 706 t.mTestName = "MediaScannerBenchmark"; 707 t.mKey = "fio_media_scanner_benchmark"; 708 709 TestFileInfo f; 710 for (int i = 0; i < mMediaScannerMediaFileCount; i++) { 711 f = new TestFileInfo(); 712 f.mFileName = new File(mMediaTestDir, String.format(mediaFile, i)).getAbsolutePath(); 713 f.mSize = mMediaScannerMediaFileSize; // File size is already in kB so do nothing. 714 t.mTestFiles.add(f); 715 } 716 717 f = new TestFileInfo(); 718 f.mFileName = new File(mMediaTestDir, workerFile).getAbsolutePath(); 719 f.mSize = mMediaScannerWorkerFileSize * 1024; // File size is in MB but we want it in kB. 720 t.mTestFiles.add(f); 721 722 JobInfo j = new JobInfo(); 723 j.mJobName = "global"; 724 j.mParameters.put("directory", mMediaTestDir); 725 j.mParameters.put("fsync", "1024"); 726 j.mParameters.put("ioengine", "sync"); 727 t.mJobs.add(j); 728 729 j = new JobInfo(); 730 j.mJobName = mediaJob; 731 StringBuilder fileNames = new StringBuilder(); 732 fileNames.append(String.format(mediaFile, 0)); 733 for (int i = 1; i < mMediaScannerMediaFileCount; i++) { 734 fileNames.append(String.format(":%s", String.format(mediaFile, i))); 735 } 736 j.mParameters.put("filename", fileNames.toString()); 737 j.mParameters.put("exitall", null); 738 j.mParameters.put("openfiles", "4"); 739 j.mParameters.put("rw", "read"); 740 t.mJobs.add(j); 741 742 j = new JobInfo(); 743 j.mJobName = workerJob; 744 j.mParameters.put("filename", workerFile); 745 j.mParameters.put("group_reporting", null); 746 j.mParameters.put("new_group", null); 747 j.mParameters.put("numjobs", String.format("%d", mMediaScannerWorkerJobCount)); 748 j.mParameters.put("rw", "randrw"); 749 j.mParameters.put("size", String.format("%dM", mMediaScannerWorkerFileSize)); 750 t.mJobs.add(j); 751 752 PerfMetricInfo m = new PerfMetricInfo(); 753 m.mJobName = mediaJob; 754 m.mPostKey = "media_bandwidth"; 755 m.mFieldName = "read-bandwidth-mean"; 756 m.mType = PerfMetricInfo.ResultType.FLOAT; 757 t.mPerfMetrics.add(m); 758 759 m = new PerfMetricInfo(); 760 m.mJobName = mediaJob; 761 m.mPostKey = "media_latency"; 762 m.mFieldName = "read-clat-mean"; 763 m.mType = PerfMetricInfo.ResultType.FLOAT; 764 t.mPerfMetrics.add(m); 765 766 m = new PerfMetricInfo(); 767 m.mJobName = workerJob; 768 m.mPostKey = "workers_read_bandwidth"; 769 m.mFieldName = "read-bandwidth-mean"; 770 m.mType = PerfMetricInfo.ResultType.FLOAT; 771 t.mPerfMetrics.add(m); 772 773 m = new PerfMetricInfo(); 774 m.mJobName = workerJob; 775 m.mPostKey = "workers_write_bandwidth"; 776 m.mFieldName = "write-bandwidth-mean"; 777 m.mType = PerfMetricInfo.ResultType.FLOAT; 778 t.mPerfMetrics.add(m); 779 780 mTestCases.add(t); 781 } 782 783 /** 784 * Creates the directories needed to run FIO, pushes the FIO executable to the device, and 785 * stops the runtime. 786 * 787 * @throws DeviceNotAvailableException if the device is not available. 788 */ 789 private void setupDevice() throws DeviceNotAvailableException { 790 mTestDevice.executeShellCommand("stop"); 791 mTestDevice.executeShellCommand(String.format("mkdir -p %s", mFioDir)); 792 mTestDevice.executeShellCommand(String.format("mkdir -p %s", mTmpDir)); 793 mTestDevice.executeShellCommand(String.format("mkdir -p %s", mInternalTestDir)); 794 mTestDevice.executeShellCommand(String.format("mkdir -p %s", mMediaTestDir)); 795 if (mExternalTestDir != null) { 796 mTestDevice.executeShellCommand(String.format("mkdir -p %s", mExternalTestDir)); 797 } 798 if (mFioLocation != null) { 799 mTestDevice.pushFile(new File(mFioLocation), mFioBin); 800 mTestDevice.executeShellCommand(String.format("chmod 755 %s", mFioBin)); 801 } 802 } 803 804 /** 805 * Reverses the actions of {@link #setDevice(ITestDevice)}. 806 * 807 * @throws DeviceNotAvailableException If the device is not available. 808 */ 809 private void cleanupDevice() throws DeviceNotAvailableException { 810 if (mExternalTestDir != null) { 811 mTestDevice.executeShellCommand(String.format("rm -r %s", mExternalTestDir)); 812 } 813 mTestDevice.executeShellCommand(String.format("rm -r %s", mMediaTestDir)); 814 mTestDevice.executeShellCommand(String.format("rm -r %s", mInternalTestDir)); 815 mTestDevice.executeShellCommand(String.format("rm -r %s", mTmpDir)); 816 mTestDevice.executeShellCommand(String.format("rm -r %s", mFioDir)); 817 mTestDevice.executeShellCommand("start"); 818 mTestDevice.waitForDeviceAvailable(); 819 } 820 821 /** 822 * Runs a single test, including creating the test files, clearing the cache, collecting before 823 * and after files, running the benchmark, and reporting the results. 824 * 825 * @param test the benchmark. 826 * @param listener the ITestInvocationListener 827 * @throws DeviceNotAvailableException if the device is not available. 828 */ 829 private void runTest(TestInfo test, ITestInvocationListener listener) 830 throws DeviceNotAvailableException { 831 CLog.i("Running %s benchmark", test.mTestName); 832 mTestDevice.executeShellCommand(String.format("rm -r %s/*", mTmpDir)); 833 mTestDevice.executeShellCommand(String.format("rm -r %s/*", mInternalTestDir)); 834 mTestDevice.executeShellCommand(String.format("rm -r %s/*", mMediaTestDir)); 835 if (mExternalTestDir != null) { 836 mTestDevice.executeShellCommand(String.format("rm -r %s/*", mExternalTestDir)); 837 } 838 839 for (TestFileInfo file : test.mTestFiles) { 840 CLog.v("Creating file: %s, size: %dkB", file.mFileName, file.mSize); 841 String cmd = String.format("dd if=/dev/urandom of=%s bs=1024 count=%d", file.mFileName, 842 file.mSize); 843 int timeout = file.mSize * 2 * 1000; // Timeout is 2 seconds per kB. 844 mTestDevice.executeShellCommand(cmd, new NullOutputReceiver(), 845 timeout, TimeUnit.MILLISECONDS, 2); 846 } 847 848 CLog.i("Creating config"); 849 CLog.d("Config file:\n%s", test.createConfig()); 850 mTestDevice.pushString(test.createConfig(), mFioConfig); 851 852 CLog.i("Dropping cache"); 853 mTestDevice.executeShellCommand("echo 3 > /proc/sys/vm/drop_caches"); 854 855 collectLogs(test, listener, "before"); 856 857 CLog.i("Running test"); 858 FioParser output = new FioParser(); 859 // Run FIO with a timeout of 1 hour. 860 mTestDevice.executeShellCommand(String.format("%s --minimal %s", mFioBin, mFioConfig), 861 output, 60 * 60 * 1000, TimeUnit.MILLISECONDS, 2); 862 863 collectLogs(test, listener, "after"); 864 865 // Report metrics 866 Map<String, String> metrics = new HashMap<>(); 867 String key = mKeySuffix == null ? test.mKey : test.mKey + mKeySuffix; 868 869 listener.testRunStarted(key, 0); 870 for (PerfMetricInfo m : test.mPerfMetrics) { 871 if (!output.mResults.containsKey(m.mJobName)) { 872 CLog.w("Job name %s was not found in the results", m.mJobName); 873 continue; 874 } 875 876 String value = output.getResult(m.mJobName, m.mFieldName); 877 if (value != null) { 878 metrics.put(m.mPostKey, m.mType.value(value)); 879 } else { 880 CLog.w("%s was not in results for the job %s", m.mFieldName, m.mJobName); 881 } 882 } 883 884 CLog.d("About to report metrics to %s: %s", key, metrics); 885 listener.testRunEnded(0, metrics); 886 } 887 888 private void collectLogs(TestInfo testInfo, ITestInvocationListener listener, 889 String descriptor) throws DeviceNotAvailableException { 890 if (mCollectYaffsLogs && mTestDevice.doesFileExist("/proc/yaffs")) { 891 logFile("/proc/yaffs", String.format("%s-yaffs-%s", testInfo.mTestName, descriptor), 892 mTestDevice, listener); 893 } 894 } 895 896 private void logFile(String remoteFileName, String localFileName, ITestDevice testDevice, 897 ITestInvocationListener listener) throws DeviceNotAvailableException { 898 File outputFile = null; 899 InputStreamSource outputSource = null; 900 try { 901 outputFile = testDevice.pullFile(remoteFileName); 902 if (outputFile != null) { 903 CLog.d("Sending %d byte file %s to logosphere!", outputFile.length(), outputFile); 904 outputSource = new FileInputStreamSource(outputFile); 905 listener.testLog(localFileName, LogDataType.TEXT, outputSource); 906 } 907 } finally { 908 FileUtil.deleteFile(outputFile); 909 StreamUtil.cancel(outputSource); 910 } 911 } 912 913 /** 914 * {@inheritDoc} 915 */ 916 @Override 917 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 918 Assert.assertNotNull(mTestDevice); 919 920 mFioDir = new File(mTestDevice.getMountPoint(IDevice.MNT_DATA), "fio").getAbsolutePath(); 921 if (mFioLocation != null) { 922 mFioBin = new File(mFioDir, "fio").getAbsolutePath(); 923 } else { 924 mFioBin = "fio"; 925 } 926 mFioConfig = new File(mFioDir, "config.fio").getAbsolutePath(); 927 928 setupTests(); 929 setupDevice(); 930 931 for (TestInfo test : mTestCases) { 932 runTest(test, listener); 933 } 934 935 cleanupDevice(); 936 } 937 938 /** 939 * {@inheritDoc} 940 */ 941 @Override 942 public void setDevice(ITestDevice device) { 943 mTestDevice = device; 944 } 945 946 /** 947 * {@inheritDoc} 948 */ 949 @Override 950 public ITestDevice getDevice() { 951 return mTestDevice; 952 } 953 954 /** 955 * A meta-test to ensure that the bits of FioBenchmarkTest are working properly. 956 */ 957 public static class MetaTest extends TestCase { 958 959 /** 960 * Test that {@link JobInfo#createJob()} properly formats a job. 961 */ 962 public void testCreateJob() { 963 JobInfo j = new JobInfo(); 964 assertEquals("", j.createJob()); 965 j.mJobName = "job"; 966 assertEquals("[job]\n", j.createJob()); 967 j.mParameters.put("param1", null); 968 j.mParameters.put("param2", "value"); 969 String[] lines = j.createJob().split("\n"); 970 assertEquals(3, lines.length); 971 assertEquals("[job]", lines[0]); 972 Set<String> params = new HashSet<>(2); 973 params.add(lines[1]); 974 params.add(lines[2]); 975 assertTrue(params.contains("param1")); 976 assertTrue(params.contains("param2=value")); 977 } 978 979 /** 980 * Test that {@link TestInfo#createConfig()} properly formats a config. 981 */ 982 public void testCreateConfig() { 983 TestInfo t = new TestInfo(); 984 JobInfo j = new JobInfo(); 985 j.mJobName = "job1"; 986 j.mParameters.put("param1", "value1"); 987 t.mJobs.add(j); 988 989 j = new JobInfo(); 990 j.mJobName = "job2"; 991 j.mParameters.put("param2", "value2"); 992 t.mJobs.add(j); 993 994 j = new JobInfo(); 995 j.mJobName = "job3"; 996 j.mParameters.put("param3", "value3"); 997 t.mJobs.add(j); 998 999 assertEquals("[job1]\nparam1=value1\n\n" + 1000 "[job2]\nparam2=value2\n\n" + 1001 "[job3]\nparam3=value3\n\n", t.createConfig()); 1002 } 1003 1004 /** 1005 * Test that output lines are parsed correctly by the FioParser, invalid lines are ignored, 1006 * and that the various fields are accessible with 1007 * {@link FioParser#getResult(String, String)}. 1008 */ 1009 public void testFioParser() { 1010 String[] lines = new String[4]; 1011 // We build the lines up as follows (assuming FIO_RESULTS_FIELDS.length == 58): 1012 // 0;3;6;...;171 1013 // 1;4;7;...;172 1014 // 2;5;8;...;173 1015 for (int i = 0; i < 3; i++) { 1016 StringBuilder sb = new StringBuilder(); 1017 sb.append(i); 1018 for (int j = 1; j < FIO_V0_RESULT_FIELDS.length; j++) { 1019 sb.append(";"); 1020 sb.append(j * 3 + i); 1021 } 1022 lines[i] = sb.toString(); 1023 } 1024 // A line may have an optional description on the end which we don't care about, make 1025 // sure it still parses. 1026 lines[2] = lines[2] += ";description"; 1027 // Make sure an invalid output line does not parse. 1028 lines[3] = "invalid"; 1029 1030 FioParser p = new FioParser(); 1031 p.processNewLines(lines); 1032 1033 1034 for (int i = 0; i < 3; i++) { 1035 for (int j = 0; j < FIO_V0_RESULT_FIELDS.length; j++) { 1036 assertEquals(String.format("job=%d, field=%s", i, FIO_V0_RESULT_FIELDS[j]), 1037 String.format("%d", j * 3 + i), 1038 p.getResult(String.format("%d", i), FIO_V0_RESULT_FIELDS[j])); 1039 } 1040 } 1041 assertNull(p.getResult("missing", "jobname")); 1042 assertNull(p.getResult("invalid", "jobname")); 1043 assertNull(p.getResult("0", "missing")); 1044 } 1045 1046 /** 1047 * Test that {@link PerfMetricInfo.ResultType#value(String)} correctly transforms strings 1048 * based on the result type. 1049 */ 1050 public void testResultTypeValue() { 1051 assertEquals("test", PerfMetricInfo.ResultType.STRING.value("test")); 1052 assertEquals("1", PerfMetricInfo.ResultType.INT.value("1")); 1053 assertEquals("3.14159", PerfMetricInfo.ResultType.FLOAT.value("3.14159")); 1054 assertEquals(String.format("%f", 0.34567), 1055 PerfMetricInfo.ResultType.PERCENT.value("34.567%")); 1056 assertNull(PerfMetricInfo.ResultType.PERCENT.value("")); 1057 assertNull(PerfMetricInfo.ResultType.PERCENT.value("34.567")); 1058 assertNull(PerfMetricInfo.ResultType.PERCENT.value("test%")); 1059 } 1060 } 1061 } 1062
