1 /** 2 * Copyright 2016 Google Inc. All Rights Reserved. 3 * 4 * <p>Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 5 * except in compliance with the License. You may obtain a copy of the License at 6 * 7 * <p>http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * <p>Unless required by applicable law or agreed to in writing, software distributed under the 10 * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 * express or implied. See the License for the specific language governing permissions and 12 * limitations under the License. 13 */ 14 package com.android.vts.util; 15 16 import com.android.vts.entity.ProfilingPointEntity; 17 import com.android.vts.entity.ProfilingPointRunEntity; 18 import com.android.vts.entity.ProfilingPointSummaryEntity; 19 import com.android.vts.proto.VtsReportMessage.VtsProfilingRegressionMode; 20 import com.google.appengine.api.datastore.DatastoreService; 21 import com.google.appengine.api.datastore.DatastoreServiceFactory; 22 import com.google.appengine.api.datastore.Entity; 23 import com.google.appengine.api.datastore.Query; 24 import java.io.IOException; 25 import java.math.RoundingMode; 26 import java.text.DecimalFormat; 27 import java.util.ArrayList; 28 import java.util.HashMap; 29 import java.util.List; 30 import java.util.Map; 31 import java.util.Set; 32 import java.util.concurrent.TimeUnit; 33 import java.util.logging.Logger; 34 import org.apache.commons.lang.StringUtils; 35 36 /** PerformanceUtil, a helper class for analyzing profiling and performance data. */ 37 public class PerformanceUtil { 38 protected static Logger logger = Logger.getLogger(PerformanceUtil.class.getName()); 39 40 private static final int MAX_BATCH_SIZE = 2000; 41 private static final DecimalFormat FORMATTER; 42 private static final String NAME_DELIMITER = ", "; 43 private static final String OPTION_DELIMITER = "="; 44 45 /** Initialize the decimal formatter. */ 46 static { 47 FORMATTER = new DecimalFormat("#.##"); 48 FORMATTER.setRoundingMode(RoundingMode.HALF_UP); 49 } 50 51 /** 52 * Creates the HTML for a table cell representing the percent change between two numbers. 53 * 54 * <p>Computes the percent change (after - before)/before * 100 and inserts it into a table cell 55 * with the specified style. The color of the cell is white if 'after' is less than before. 56 * Otherwise, the cell is colored red with opacity according to the percent change (100%+ delta 57 * means 100% opacity). If the before value is 0 and the after value is positive, then the color 58 * of the cell is 100% red to indicate an increase of undefined magnitude. 59 * 60 * @param baseline The baseline value observed. 61 * @param test The value to compare against the baseline. 62 * @param classNames A string containing HTML classes to apply to the table cell. 63 * @param style A string containing additional CSS styles. 64 * @returns An HTML string for a colored table cell containing the percent change. 65 */ 66 public static String getPercentChangeHTML( 67 double baseline, 68 double test, 69 String classNames, 70 String style, 71 VtsProfilingRegressionMode mode) { 72 String pctChangeString = "0 %"; 73 double alpha = 0; 74 double delta = test - baseline; 75 if (baseline != 0) { 76 double pctChange = delta / baseline; 77 alpha = pctChange * 2; 78 pctChangeString = FORMATTER.format(pctChange * 100) + " %"; 79 } else if (delta != 0) { 80 // If the percent change is undefined, the cell will be solid red or white 81 alpha = (int) Math.signum(delta); // get the sign of the delta (+1, 0, -1) 82 pctChangeString = ""; 83 } 84 if (mode == VtsProfilingRegressionMode.VTS_REGRESSION_MODE_DECREASING) { 85 alpha = -alpha; 86 } 87 String color = "background-color: rgba(255, 0, 0, " + alpha + "); "; 88 String html = "<td class='" + classNames + "' style='" + color + style + "'>"; 89 html += pctChangeString + "</td>"; 90 return html; 91 } 92 93 /** 94 * Compares a test StatSummary to a baseline StatSummary using best-case performance. 95 * 96 * @param baseline The StatSummary object containing initial values to compare against 97 * @param test The StatSummary object containing test values to be compared against the baseline 98 * @param innerClasses Class names to apply to cells on the inside of the grid 99 * @param outerClasses Class names to apply to cells on the outside of the grid 100 * @param innerStyles CSS styles to apply to cells on the inside of the grid 101 * @param outerStyles CSS styles to apply to cells on the outside of the grid 102 * @return HTML string representing the performance of the test versus the baseline 103 */ 104 public static String getBestCasePerformanceComparisonHTML( 105 StatSummary baseline, 106 StatSummary test, 107 String innerClasses, 108 String outerClasses, 109 String innerStyles, 110 String outerStyles) { 111 if (test == null || baseline == null) { 112 return "<td></td><td></td><td></td><td></td>"; 113 } 114 String row = ""; 115 // Intensity of red color is a function of the relative (percent) change 116 // in the new value compared to the previous day's. Intensity is a linear function 117 // of percentage change, reaching a ceiling at 100% change (e.g. a doubling). 118 row += 119 getPercentChangeHTML( 120 baseline.getBestCase(), 121 test.getBestCase(), 122 innerClasses, 123 innerStyles, 124 test.getRegressionMode()); 125 row += "<td class='" + innerClasses + "' style='" + innerStyles + "'>"; 126 row += FORMATTER.format(baseline.getBestCase()); 127 row += "<td class='" + innerClasses + "' style='" + innerStyles + "'>"; 128 row += FORMATTER.format(baseline.getMean()); 129 row += "<td class='" + outerClasses + "' style='" + outerStyles + "'>"; 130 row += FORMATTER.format(baseline.getStd()) + "</td>"; 131 return row; 132 } 133 134 /** 135 * Updates a PerformanceSummary object with data in the specified window. 136 * 137 * @param testName The name of the table whose profiling vectors to retrieve. 138 * @param startTime The (inclusive) start time in microseconds to scan from. 139 * @param endTime The (inclusive) end time in microseconds at which to stop scanning. 140 * @param selectedDevice The name of the device whose data to query for, or null for unfiltered. 141 * @param summaries The list of PerformanceSummary objects to populate with data. 142 * @throws IOException 143 */ 144 public static void updatePerformanceSummary( 145 String testName, 146 long startTime, 147 long endTime, 148 String selectedDevice, 149 List<PerformanceSummary> summaries) { 150 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 151 Query profilingPointQuery = 152 new Query(ProfilingPointEntity.KIND) 153 .setFilter( 154 new Query.FilterPredicate( 155 ProfilingPointEntity.TEST_NAME, 156 Query.FilterOperator.EQUAL, 157 testName)); 158 159 List<ProfilingPointEntity> profilingPoints = new ArrayList<>(); 160 for (Entity e : 161 datastore 162 .prepare(profilingPointQuery) 163 .asIterable(DatastoreHelper.getLargeBatchOptions())) { 164 ProfilingPointEntity pp = ProfilingPointEntity.fromEntity(e); 165 if (pp == null) continue; 166 profilingPoints.add(pp); 167 } 168 169 Query.Filter startFilter = 170 new Query.FilterPredicate( 171 ProfilingPointSummaryEntity.START_TIME, 172 Query.FilterOperator.GREATER_THAN_OR_EQUAL, 173 startTime); 174 Query.Filter endFilter = 175 new Query.FilterPredicate( 176 ProfilingPointSummaryEntity.START_TIME, 177 Query.FilterOperator.LESS_THAN_OR_EQUAL, 178 endTime); 179 Query.Filter timeFilter = Query.CompositeFilterOperator.and(startFilter, endFilter); 180 181 Query.Filter deviceFilter; 182 if (selectedDevice != null) { 183 deviceFilter = FilterUtil.FilterKey.TARGET.getFilterForString(selectedDevice); 184 } else { 185 deviceFilter = 186 FilterUtil.FilterKey.TARGET.getFilterForString(ProfilingPointSummaryEntity.ALL); 187 } 188 deviceFilter = 189 Query.CompositeFilterOperator.and( 190 deviceFilter, 191 FilterUtil.FilterKey.BRANCH.getFilterForString( 192 ProfilingPointSummaryEntity.ALL)); 193 Query.Filter filter = Query.CompositeFilterOperator.and(timeFilter, deviceFilter); 194 195 Map<ProfilingPointEntity, Iterable<Entity>> asyncEntities = new HashMap<>(); 196 for (ProfilingPointEntity pp : profilingPoints) { 197 Query profilingQuery = 198 new Query(ProfilingPointSummaryEntity.KIND) 199 .setAncestor(pp.key) 200 .setFilter(filter); 201 asyncEntities.put( 202 pp, 203 datastore 204 .prepare(profilingQuery) 205 .asIterable(DatastoreHelper.getLargeBatchOptions())); 206 } 207 208 for (ProfilingPointEntity pp : asyncEntities.keySet()) { 209 for (Entity ppSummaryEntity : asyncEntities.get(pp)) { 210 ProfilingPointSummaryEntity ppSummary = 211 ProfilingPointSummaryEntity.fromEntity(ppSummaryEntity); 212 if (ppSummary == null) continue; 213 for (PerformanceSummary perfSummary : summaries) { 214 if (perfSummary.contains(ppSummary.startTime)) { 215 perfSummary.addData(pp, ppSummaryEntity); 216 } 217 } 218 } 219 } 220 } 221 222 /** 223 * Compares a test StatSummary to a baseline StatSummary using average-case performance. 224 * 225 * @param baseline The StatSummary object containing initial values to compare against 226 * @param test The StatSummary object containing test values to be compared against the baseline 227 * @param innerClasses Class names to apply to cells on the inside of the grid 228 * @param outerClasses Class names to apply to cells on the outside of the grid 229 * @param innerStyles CSS styles to apply to cells on the inside of the grid 230 * @param outerStyles CSS styles to apply to cells on the outside of the grid 231 * @return HTML string representing the performance of the test versus the baseline 232 */ 233 public static String getAvgCasePerformanceComparisonHTML( 234 StatSummary baseline, 235 StatSummary test, 236 String innerClasses, 237 String outerClasses, 238 String innerStyles, 239 String outerStyles) { 240 if (test == null || baseline == null) { 241 return "<td></td><td></td><td></td><td></td>"; 242 } 243 String row = ""; 244 // Intensity of red color is a function of the relative (percent) change 245 // in the new value compared to the previous day's. Intensity is a linear function 246 // of percentage change, reaching a ceiling at 100% change (e.g. a doubling). 247 row += 248 getPercentChangeHTML( 249 baseline.getMean(), 250 test.getMean(), 251 innerClasses, 252 innerStyles, 253 test.getRegressionMode()); 254 row += "<td class='" + innerClasses + "' style='" + innerStyles + "'>"; 255 row += FORMATTER.format(baseline.getBestCase()); 256 row += "<td class='" + innerClasses + "' style='" + innerStyles + "'>"; 257 row += FORMATTER.format(baseline.getMean()); 258 row += "<td class='" + outerClasses + "' style='" + outerStyles + "'>"; 259 row += FORMATTER.format(baseline.getStd()) + "</td>"; 260 return row; 261 } 262 263 /** 264 * Generates a string of the values in optionsList which have matches in the profiling entity. 265 * 266 * @param profilingRun The entity for a profiling point run. 267 * @param optionKeys A list of keys to match against the optionsList key value pairs. 268 * @return The values in optionsList whose key match a key in optionKeys. 269 */ 270 public static String getOptionAlias( 271 ProfilingPointRunEntity profilingRun, Set<String> optionKeys) { 272 String name = ""; 273 if (profilingRun.options != null) { 274 name = getOptionAlias(profilingRun.options, optionKeys); 275 } 276 return name; 277 } 278 279 /** 280 * Generates a string of the values in optionsList which have matches in the profiling entity. 281 * 282 * @param optionList The list of key=value option pair strings. 283 * @param optionKeys A list of keys to match against the optionsList key value pairs. 284 * @return The values in optionsList whose key match a key in optionKeys. 285 */ 286 public static String getOptionAlias(List<String> optionList, Set<String> optionKeys) { 287 String name = ""; 288 List<String> nameSuffixes = new ArrayList<>(); 289 for (String optionString : optionList) { 290 String[] optionParts = optionString.split(OPTION_DELIMITER); 291 if (optionParts.length != 2) { 292 continue; 293 } 294 if (optionKeys.contains(optionParts[0].trim().toLowerCase())) { 295 nameSuffixes.add(optionParts[1].trim().toLowerCase()); 296 } 297 } 298 if (nameSuffixes.size() > 0) { 299 StringUtils.join(nameSuffixes, NAME_DELIMITER); 300 name += StringUtils.join(nameSuffixes, NAME_DELIMITER); 301 } 302 return name; 303 } 304 } 305