Home | History | Annotate | Download | only in results 
      1 /*
      2  * Copyright (C) 2015 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the License);
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an AS IS BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 package com.android.benchmark.results;
     18 
     19 import android.annotation.TargetApi;
     20 import android.view.FrameMetrics;
     21 
     22 import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
     23 import org.apache.commons.math.stat.descriptive.SummaryStatistics;
     24 
     25 import java.util.ArrayList;
     26 import java.util.List;
     27 
     28 /**
     29  * Utility for storing and analyzing UI benchmark results.
     30  */
     31 @TargetApi(24)
     32 public class UiBenchmarkResult {
     33     private static final int BASE_SCORE = 100;
     34     private static final int ZERO_SCORE_TOTAL_DURATION_MS = 32;
     35     private static final int JANK_PENALTY_THRESHOLD_MS = 12;
     36     private static final int ZERO_SCORE_ABOVE_THRESHOLD_MS =
     37             ZERO_SCORE_TOTAL_DURATION_MS - JANK_PENALTY_THRESHOLD_MS;
     38     private static final double JANK_PENALTY_PER_MS_ABOVE_THRESHOLD =
     39             BASE_SCORE / (double)ZERO_SCORE_ABOVE_THRESHOLD_MS;
     40     private static final int CONSISTENCY_BONUS_MAX = 100;
     41 
     42     private static final int METRIC_WAS_JANKY = -1;
     43 
     44     private static final int[] METRICS = new int[] {
     45             FrameMetrics.UNKNOWN_DELAY_DURATION,
     46             FrameMetrics.INPUT_HANDLING_DURATION,
     47             FrameMetrics.ANIMATION_DURATION,
     48             FrameMetrics.LAYOUT_MEASURE_DURATION,
     49             FrameMetrics.DRAW_DURATION,
     50             FrameMetrics.SYNC_DURATION,
     51             FrameMetrics.COMMAND_ISSUE_DURATION,
     52             FrameMetrics.SWAP_BUFFERS_DURATION,
     53             FrameMetrics.TOTAL_DURATION,
     54     };
     55     public static final int FRAME_PERIOD_MS = 16;
     56 
     57     private final DescriptiveStatistics[] mStoredStatistics;
     58 
     59     public UiBenchmarkResult(List<FrameMetrics> instances) {
     60         mStoredStatistics = new DescriptiveStatistics[METRICS.length];
     61         insertMetrics(instances);
     62     }
     63 
     64     public UiBenchmarkResult(double[] values) {
     65         mStoredStatistics = new DescriptiveStatistics[METRICS.length];
     66         insertValues(values);
     67     }
     68 
     69     public void update(List<FrameMetrics> instances) {
     70         insertMetrics(instances);
     71     }
     72 
     73     public void update(double[] values) {
     74         insertValues(values);
     75     }
     76 
     77     public double getAverage(int id) {
     78         int pos = getMetricPosition(id);
     79         return mStoredStatistics[pos].getMean();
     80     }
     81 
     82     public double getMinimum(int id) {
     83         int pos = getMetricPosition(id);
     84         return mStoredStatistics[pos].getMin();
     85     }
     86 
     87     public double getMaximum(int id) {
     88         int pos = getMetricPosition(id);
     89         return mStoredStatistics[pos].getMax();
     90     }
     91 
     92     public int getMaximumIndex(int id) {
     93         int pos = getMetricPosition(id);
     94         double[] storedMetrics = mStoredStatistics[pos].getValues();
     95         int maxIdx = 0;
     96         for (int i = 0; i < storedMetrics.length; i++) {
     97             if (storedMetrics[i] >= storedMetrics[maxIdx]) {
     98                 maxIdx = i;
     99             }
    100         }
    101 
    102         return maxIdx;
    103     }
    104 
    105     public double getMetricAtIndex(int index, int metricId) {
    106         return mStoredStatistics[getMetricPosition(metricId)].getElement(index);
    107     }
    108 
    109     public double getPercentile(int id, int percentile) {
    110         if (percentile > 100) percentile = 100;
    111         if (percentile < 0) percentile = 0;
    112 
    113         int metricPos = getMetricPosition(id);
    114         return mStoredStatistics[metricPos].getPercentile(percentile);
    115     }
    116 
    117     public int getTotalFrameCount() {
    118         if (mStoredStatistics.length == 0) {
    119             return 0;
    120         }
    121 
    122         return (int) mStoredStatistics[0].getN();
    123     }
    124 
    125     public int getScore() {
    126         SummaryStatistics badFramesStats = new SummaryStatistics();
    127 
    128         int totalFrameCount = getTotalFrameCount();
    129         for (int i = 0; i < totalFrameCount; i++) {
    130             double totalDuration = getMetricAtIndex(i, FrameMetrics.TOTAL_DURATION);
    131             if (totalDuration >= 12) {
    132                 badFramesStats.addValue(totalDuration);
    133             }
    134         }
    135 
    136         int length = getSortedJankFrameIndices().length;
    137         double jankFrameCount = 100 * length / (double) totalFrameCount;
    138 
    139         System.out.println("Mean: " + badFramesStats.getMean() + " JankP: " + jankFrameCount
    140                 + " StdDev: " + badFramesStats.getStandardDeviation() +
    141                 " Count Bad: " + badFramesStats.getN() + " Count Jank: " + length);
    142 
    143         return (int) Math.round(
    144                 (badFramesStats.getMean()) * jankFrameCount * badFramesStats.getStandardDeviation());
    145     }
    146 
    147     public int getJankPenalty() {
    148         double total95th = mStoredStatistics[getMetricPosition(FrameMetrics.TOTAL_DURATION)]
    149                 .getPercentile(95);
    150         System.out.println("95: " + total95th);
    151         double aboveThreshold = total95th - JANK_PENALTY_THRESHOLD_MS;
    152         if (aboveThreshold <= 0) {
    153             return 0;
    154         }
    155 
    156         if (aboveThreshold > ZERO_SCORE_ABOVE_THRESHOLD_MS) {
    157             return BASE_SCORE;
    158         }
    159 
    160         return (int) Math.ceil(JANK_PENALTY_PER_MS_ABOVE_THRESHOLD * aboveThreshold);
    161     }
    162 
    163     public int getConsistencyBonus() {
    164         DescriptiveStatistics totalDurationStats =
    165                 mStoredStatistics[getMetricPosition(FrameMetrics.TOTAL_DURATION)];
    166 
    167         double standardDeviation = totalDurationStats.getStandardDeviation();
    168         if (standardDeviation == 0) {
    169             return CONSISTENCY_BONUS_MAX;
    170         }
    171 
    172         // 1 / CV of the total duration.
    173         double bonus = totalDurationStats.getMean() / standardDeviation;
    174         return (int) Math.min(Math.round(bonus), CONSISTENCY_BONUS_MAX);
    175     }
    176 
    177     public int[] getSortedJankFrameIndices() {
    178         ArrayList<Integer> jankFrameIndices = new ArrayList<>();
    179         boolean tripleBuffered = false;
    180         int totalFrameCount = getTotalFrameCount();
    181         int totalDurationPos = getMetricPosition(FrameMetrics.TOTAL_DURATION);
    182 
    183         for (int i = 0; i < totalFrameCount; i++) {
    184             double thisDuration = mStoredStatistics[totalDurationPos].getElement(i);
    185             if (!tripleBuffered) {
    186                 if (thisDuration > FRAME_PERIOD_MS) {
    187                     tripleBuffered = true;
    188                     jankFrameIndices.add(i);
    189                 }
    190             } else {
    191                 if (thisDuration > 2 * FRAME_PERIOD_MS) {
    192                     tripleBuffered = false;
    193                     jankFrameIndices.add(i);
    194                 }
    195             }
    196         }
    197 
    198         int[] res = new int[jankFrameIndices.size()];
    199         int i = 0;
    200         for (Integer index : jankFrameIndices) {
    201             res[i++] = index;
    202         }
    203         return res;
    204     }
    205 
    206     private int getMetricPosition(int id) {
    207         for (int i = 0; i < METRICS.length; i++) {
    208             if (id == METRICS[i]) {
    209                 return i;
    210             }
    211         }
    212 
    213         return -1;
    214     }
    215 
    216     private void insertMetrics(List<FrameMetrics> instances) {
    217         for (FrameMetrics frame : instances) {
    218             for (int i = 0; i < METRICS.length; i++) {
    219                 DescriptiveStatistics stats = mStoredStatistics[i];
    220                 if (stats == null) {
    221                     stats = new DescriptiveStatistics();
    222                     mStoredStatistics[i] = stats;
    223                 }
    224 
    225                 mStoredStatistics[i].addValue(frame.getMetric(METRICS[i]) / (double) 1000000);
    226             }
    227         }
    228     }
    229 
    230     private void insertValues(double[] values) {
    231         if (values.length != METRICS.length) {
    232             throw new IllegalArgumentException("invalid values array");
    233         }
    234 
    235         for (int i = 0; i < values.length; i++) {
    236             DescriptiveStatistics stats = mStoredStatistics[i];
    237             if (stats == null) {
    238                 stats = new DescriptiveStatistics();
    239                 mStoredStatistics[i] = stats;
    240             }
    241 
    242             mStoredStatistics[i].addValue(values[i]);
    243         }
    244     }
    245  }
    246