1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.perftests.utils; 18 19 import android.app.Activity; 20 import android.app.Instrumentation; 21 import android.os.Bundle; 22 import android.util.Log; 23 24 import java.util.ArrayList; 25 import java.util.concurrent.TimeUnit; 26 27 /** 28 * Provides a benchmark framework. 29 * 30 * This differs from BenchmarkState in that rather than the class measuring the the elapsed time, 31 * the test passes in the elapsed time. 32 * 33 * Example usage: 34 * 35 * public void sampleMethod() { 36 * ManualBenchmarkState state = new ManualBenchmarkState(); 37 * 38 * int[] src = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; 39 * long elapsedTime = 0; 40 * while (state.keepRunning(elapsedTime)) { 41 * long startTime = System.nanoTime(); 42 * int[] dest = new int[src.length]; 43 * System.arraycopy(src, 0, dest, 0, src.length); 44 * elapsedTime = System.nanoTime() - startTime; 45 * } 46 * System.out.println(state.summaryLine()); 47 * } 48 * 49 * Or use the PerfManualStatusReporter TestRule. 50 * 51 * Make sure that the overhead of checking the clock does not noticeably affect the results. 52 */ 53 public final class ManualBenchmarkState { 54 private static final String TAG = ManualBenchmarkState.class.getSimpleName(); 55 56 // TODO: Tune these values. 57 // warm-up for duration 58 private static final long WARMUP_DURATION_NS = TimeUnit.SECONDS.toNanos(5); 59 // minimum iterations to warm-up for 60 private static final int WARMUP_MIN_ITERATIONS = 8; 61 62 // target testing for duration 63 private static final long TARGET_TEST_DURATION_NS = TimeUnit.SECONDS.toNanos(16); 64 private static final int MAX_TEST_ITERATIONS = 1000000; 65 private static final int MIN_TEST_ITERATIONS = 10; 66 67 private static final int NOT_STARTED = 0; // The benchmark has not started yet. 68 private static final int WARMUP = 1; // The benchmark is warming up. 69 private static final int RUNNING = 2; // The benchmark is running. 70 private static final int FINISHED = 3; // The benchmark has stopped. 71 72 private int mState = NOT_STARTED; // Current benchmark state. 73 74 private long mWarmupStartTime = 0; 75 private int mWarmupIterations = 0; 76 77 private int mMaxIterations = 0; 78 79 // Individual duration in nano seconds. 80 private ArrayList<Long> mResults = new ArrayList<>(); 81 82 // Statistics. These values will be filled when the benchmark has finished. 83 // The computation needs double precision, but long int is fine for final reporting. 84 private Stats mStats; 85 86 private void beginBenchmark(long warmupDuration, int iterations) { 87 mMaxIterations = (int) (TARGET_TEST_DURATION_NS / (warmupDuration / iterations)); 88 mMaxIterations = Math.min(MAX_TEST_ITERATIONS, 89 Math.max(mMaxIterations, MIN_TEST_ITERATIONS)); 90 mState = RUNNING; 91 } 92 93 /** 94 * Judges whether the benchmark needs more samples. 95 * 96 * For the usage, see class comment. 97 */ 98 public boolean keepRunning(long duration) { 99 if (duration < 0) { 100 throw new RuntimeException("duration is negative: " + duration); 101 } 102 switch (mState) { 103 case NOT_STARTED: 104 mState = WARMUP; 105 mWarmupStartTime = System.nanoTime(); 106 return true; 107 case WARMUP: { 108 final long timeSinceStartingWarmup = System.nanoTime() - mWarmupStartTime; 109 ++mWarmupIterations; 110 if (mWarmupIterations >= WARMUP_MIN_ITERATIONS 111 && timeSinceStartingWarmup >= WARMUP_DURATION_NS) { 112 beginBenchmark(timeSinceStartingWarmup, mWarmupIterations); 113 } 114 return true; 115 } 116 case RUNNING: { 117 mResults.add(duration); 118 final boolean keepRunning = mResults.size() < mMaxIterations; 119 if (!keepRunning) { 120 mStats = new Stats(mResults); 121 mState = FINISHED; 122 } 123 return keepRunning; 124 } 125 case FINISHED: 126 throw new IllegalStateException("The benchmark has finished."); 127 default: 128 throw new IllegalStateException("The benchmark is in an unknown state."); 129 } 130 } 131 132 private String summaryLine() { 133 final StringBuilder sb = new StringBuilder(); 134 sb.append("Summary: "); 135 sb.append("median=").append(mStats.getMedian()).append("ns, "); 136 sb.append("mean=").append(mStats.getMean()).append("ns, "); 137 sb.append("min=").append(mStats.getMin()).append("ns, "); 138 sb.append("max=").append(mStats.getMax()).append("ns, "); 139 sb.append("sigma=").append(mStats.getStandardDeviation()).append(", "); 140 sb.append("iteration=").append(mResults.size()).append(", "); 141 sb.append("values=").append(mResults.toString()); 142 return sb.toString(); 143 } 144 145 public void sendFullStatusReport(Instrumentation instrumentation, String key) { 146 if (mState != FINISHED) { 147 throw new IllegalStateException("The benchmark hasn't finished"); 148 } 149 Log.i(TAG, key + summaryLine()); 150 final Bundle status = new Bundle(); 151 status.putLong(key + "_median", mStats.getMedian()); 152 status.putLong(key + "_mean", (long) mStats.getMean()); 153 status.putLong(key + "_percentile90", mStats.getPercentile90()); 154 status.putLong(key + "_percentile95", mStats.getPercentile95()); 155 status.putLong(key + "_stddev", (long) mStats.getStandardDeviation()); 156 instrumentation.sendStatus(Activity.RESULT_OK, status); 157 } 158 } 159 160
