1 /* 2 * Copyright (c) 2016 Google Inc. All Rights Reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you 5 * may not use this file except in compliance with the License. You may 6 * obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 * implied. See the License for the specific language governing 14 * permissions and limitations under the License. 15 */ 16 17 package com.android.vts.job; 18 19 import static org.junit.Assert.*; 20 21 import com.android.vts.entity.ProfilingPointEntity; 22 import com.android.vts.entity.ProfilingPointSummaryEntity; 23 import com.android.vts.proto.VtsReportMessage.VtsProfilingRegressionMode; 24 import com.android.vts.util.PerformanceSummary; 25 import com.android.vts.util.ProfilingPointSummary; 26 import com.android.vts.util.StatSummary; 27 import com.google.appengine.tools.development.testing.LocalDatastoreServiceTestConfig; 28 import com.google.appengine.tools.development.testing.LocalServiceTestHelper; 29 import java.io.BufferedReader; 30 import java.io.File; 31 import java.io.FileNotFoundException; 32 import java.io.FileReader; 33 import java.io.IOException; 34 import java.util.ArrayList; 35 import java.util.Arrays; 36 import java.util.HashMap; 37 import java.util.List; 38 import java.util.Map; 39 import org.junit.After; 40 import org.junit.Before; 41 import org.junit.Test; 42 43 public class VtsPerformanceJobServletTest { 44 private final LocalServiceTestHelper helper = 45 new LocalServiceTestHelper(new LocalDatastoreServiceTestConfig()); 46 47 private static final String LABEL = "testLabel"; 48 private static final String ROOT = "src/test/res/servlet"; 49 private static final String[] LABELS = new String[] {"label1", "label2", "label3"}; 50 private static final long[] HIGH_VALS = new long[] {10, 20, 30}; 51 private static final long[] LOW_VALS = new long[] {1, 2, 3}; 52 53 List<PerformanceSummary> dailySummaries; 54 List<String> legendLabels; 55 56 /** 57 * Helper method for creating ProfilingPointSummaryEntity objects. 58 * 59 * @param labels The list of data labels. 60 * @param values The list of data values. Must be equal in size to the labels list. 61 * @param regressionMode The regression mode. 62 * @return A ProfilingPointSummaryEntity with specified data. 63 */ 64 private static ProfilingPointSummaryEntity createProfilingReport( 65 String[] labels, long[] values, VtsProfilingRegressionMode regressionMode) { 66 List<String> labelList = Arrays.asList(labels); 67 StatSummary globalStats = new StatSummary("global", regressionMode); 68 Map<String, StatSummary> labelStats = new HashMap<>(); 69 for (int i = 0; i < labels.length; ++i) { 70 StatSummary stat = new StatSummary(labels[i], regressionMode); 71 stat.updateStats(values[i]); 72 labelStats.put(labels[i], stat); 73 globalStats.updateStats(values[i]); 74 } 75 return new ProfilingPointSummaryEntity( 76 ProfilingPointEntity.createKey("test", "pp"), 77 globalStats, 78 labelList, 79 labelStats, 80 "branch", 81 "build", 82 null, 83 0); 84 } 85 86 /** Asserts whether text is the same as the contents in the baseline file specified. */ 87 private static void compareToBaseline(String text, String baselineFilename) 88 throws FileNotFoundException, IOException { 89 File f = new File(ROOT, baselineFilename); 90 String baseline = ""; 91 try (BufferedReader br = new BufferedReader(new FileReader(f))) { 92 StringBuilder sb = new StringBuilder(); 93 String line = br.readLine(); 94 95 while (line != null) { 96 sb.append(line); 97 line = br.readLine(); 98 } 99 baseline = sb.toString(); 100 } 101 assertEquals(baseline, text); 102 } 103 104 @Before 105 public void setUp() { 106 helper.setUp(); 107 } 108 109 @After 110 public void tearDown() { 111 helper.tearDown(); 112 } 113 114 public void setUp(boolean grouped) { 115 dailySummaries = new ArrayList<>(); 116 legendLabels = new ArrayList<>(); 117 legendLabels.add(""); 118 119 // Add today's data 120 PerformanceSummary today = new PerformanceSummary(0, 1); 121 VtsProfilingRegressionMode mode = VtsProfilingRegressionMode.VTS_REGRESSION_MODE_INCREASING; 122 ProfilingPointSummary summary = new ProfilingPointSummary("", "", mode); 123 ProfilingPointSummaryEntity pt = createProfilingReport(LABELS, HIGH_VALS, mode); 124 if (grouped) { 125 summary.updateLabel(pt, LABEL); 126 summary.updateLabel(pt, LABEL); 127 } else { 128 summary.update(pt); 129 summary.update(pt); 130 } 131 today.insertProfilingPointSummary("p1", summary); 132 133 mode = VtsProfilingRegressionMode.VTS_REGRESSION_MODE_DECREASING; 134 summary = new ProfilingPointSummary("", "", mode); 135 pt = createProfilingReport(LABELS, LOW_VALS, mode); 136 if (grouped) { 137 summary.updateLabel(pt, LABEL); 138 summary.updateLabel(pt, LABEL); 139 } else { 140 summary.update(pt); 141 summary.update(pt); 142 } 143 today.insertProfilingPointSummary("p2", summary); 144 dailySummaries.add(today); 145 legendLabels.add("today"); 146 147 // Add yesterday data with regressions 148 PerformanceSummary yesterday = new PerformanceSummary(0, 1); 149 mode = VtsProfilingRegressionMode.VTS_REGRESSION_MODE_INCREASING; 150 summary = new ProfilingPointSummary("", "", mode); 151 pt = createProfilingReport(LABELS, LOW_VALS, mode); 152 if (grouped) { 153 summary.updateLabel(pt, LABEL); 154 summary.updateLabel(pt, LABEL); 155 } else { 156 summary.update(pt); 157 summary.update(pt); 158 } 159 yesterday.insertProfilingPointSummary("p1", summary); 160 161 mode = VtsProfilingRegressionMode.VTS_REGRESSION_MODE_DECREASING; 162 summary = new ProfilingPointSummary("x", "y", mode); 163 pt = createProfilingReport(LABELS, HIGH_VALS, mode); 164 if (grouped) { 165 summary.updateLabel(pt, LABEL); 166 summary.updateLabel(pt, LABEL); 167 } else { 168 summary.update(pt); 169 summary.update(pt); 170 } 171 yesterday.insertProfilingPointSummary("p2", summary); 172 dailySummaries.add(yesterday); 173 legendLabels.add("yesterday"); 174 175 // Add last week data without regressions 176 PerformanceSummary lastWeek = new PerformanceSummary(0, 1); 177 mode = VtsProfilingRegressionMode.VTS_REGRESSION_MODE_INCREASING; 178 summary = new ProfilingPointSummary("", "", mode); 179 pt = createProfilingReport(LABELS, HIGH_VALS, mode); 180 summary.update(pt); 181 summary.update(pt); 182 lastWeek.insertProfilingPointSummary("p1", summary); 183 184 mode = VtsProfilingRegressionMode.VTS_REGRESSION_MODE_DECREASING; 185 summary = new ProfilingPointSummary("", "", mode); 186 pt = createProfilingReport(LABELS, LOW_VALS, mode); 187 summary.update(pt); 188 summary.update(pt); 189 lastWeek.insertProfilingPointSummary("p2", summary); 190 dailySummaries.add(lastWeek); 191 legendLabels.add("last week"); 192 } 193 194 /** 195 * End-to-end test of performance report in the normal case. The normal case is when a profiling 196 * point is added or removed from the test. 197 */ 198 @Test 199 public void testPerformanceSummaryNormal() throws FileNotFoundException, IOException { 200 setUp(false); 201 String output = 202 VtsPerformanceJobServlet.getPerformanceSummary( 203 "test", dailySummaries, legendLabels); 204 compareToBaseline(output, "performanceSummary1.html"); 205 } 206 207 /** 208 * End-to-end test of performance report when a profiling point was removed in the latest run. 209 */ 210 @Test 211 public void testPerformanceSummaryDroppedProfilingPoint() 212 throws FileNotFoundException, IOException { 213 setUp(false); 214 PerformanceSummary yesterday = dailySummaries.get(dailySummaries.size() - 1); 215 VtsProfilingRegressionMode mode = VtsProfilingRegressionMode.VTS_REGRESSION_MODE_INCREASING; 216 ProfilingPointSummary summary = new ProfilingPointSummary("x", "y", mode); 217 ProfilingPointSummaryEntity pt = createProfilingReport(LABELS, HIGH_VALS, mode); 218 summary.update(pt); 219 summary.update(pt); 220 yesterday.insertProfilingPointSummary("p3", summary); 221 String output = 222 VtsPerformanceJobServlet.getPerformanceSummary( 223 "test", dailySummaries, legendLabels); 224 compareToBaseline(output, "performanceSummary2.html"); 225 } 226 227 /** End-to-end test of performance report when a profiling point was added in the latest run. */ 228 @Test 229 public void testPerformanceSummaryAddedProfilingPoint() 230 throws FileNotFoundException, IOException { 231 setUp(false); 232 PerformanceSummary today = dailySummaries.get(0); 233 VtsProfilingRegressionMode mode = VtsProfilingRegressionMode.VTS_REGRESSION_MODE_INCREASING; 234 ProfilingPointSummary summary = new ProfilingPointSummary("", "", mode); 235 ProfilingPointSummaryEntity pt = createProfilingReport(LABELS, HIGH_VALS, mode); 236 summary.update(pt); 237 summary.update(pt); 238 today.insertProfilingPointSummary("p3", summary); 239 String output = 240 VtsPerformanceJobServlet.getPerformanceSummary( 241 "test", dailySummaries, legendLabels); 242 System.out.println(output); 243 compareToBaseline(output, "performanceSummary3.html"); 244 } 245 246 /** 247 * End-to-end test of performance report labels are grouped (e.g. as if using unlabeled data) 248 */ 249 @Test 250 public void testPerformanceSummaryGroupedNormal() throws FileNotFoundException, IOException { 251 setUp(true); 252 String output = 253 VtsPerformanceJobServlet.getPerformanceSummary( 254 "test", dailySummaries, legendLabels); 255 compareToBaseline(output, "performanceSummary4.html"); 256 } 257 } 258
