1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.server.cts; 17 18 import android.service.GraphicsStatsHistogramBucketProto; 19 import android.service.GraphicsStatsJankSummaryProto; 20 import android.service.GraphicsStatsProto; 21 import android.service.GraphicsStatsServiceDumpProto; 22 23 import java.util.ArrayList; 24 import java.util.Date; 25 import java.util.List; 26 27 public class GraphicsStatsValidationTest extends ProtoDumpTestCase { 28 private static final String TAG = "GraphicsStatsValidationTest"; 29 30 private static final String DEVICE_SIDE_TEST_APK = "CtsGraphicsStatsApp.apk"; 31 private static final String DEVICE_SIDE_TEST_PACKAGE 32 = "com.android.server.cts.device.graphicsstats"; 33 34 @Override 35 protected void tearDown() throws Exception { 36 getDevice().uninstallPackage(DEVICE_SIDE_TEST_PACKAGE); 37 super.tearDown(); 38 } 39 40 @Override 41 protected void setUp() throws Exception { 42 super.setUp(); 43 installPackage(DEVICE_SIDE_TEST_APK, /* grantPermissions= */ true); 44 // Ensure that we have a starting point for our stats 45 runDeviceTests(DEVICE_SIDE_TEST_PACKAGE, ".SimpleDrawFrameTests", 46 "testDrawTenFrames"); 47 // Kill to ensure that stats persist/merge across process death 48 killTestApp(); 49 } 50 51 public void testBasicDrawFrame() throws Exception { 52 GraphicsStatsProto[] results = runDrawTest("testDrawTenFrames"); 53 GraphicsStatsProto statsBefore = results[0]; 54 GraphicsStatsProto statsAfter = results[1]; 55 GraphicsStatsJankSummaryProto summaryBefore = statsBefore.getSummary(); 56 GraphicsStatsJankSummaryProto summaryAfter = statsAfter.getSummary(); 57 assertTrue(summaryAfter.getTotalFrames() > summaryBefore.getTotalFrames()); 58 59 int frameDelta = summaryAfter.getTotalFrames() - summaryBefore.getTotalFrames(); 60 int jankyDelta = summaryAfter.getJankyFrames() - summaryBefore.getJankyFrames(); 61 // We expect 11 frames to have been drawn (first frame + the 10 more explicitly requested) 62 assertEquals(11, frameDelta); 63 assertTrue(jankyDelta < 5); 64 int veryJankyDelta = countFramesAbove(statsAfter, 40) - countFramesAbove(statsBefore, 40); 65 // The 1st frame could be >40ms, but nothing after that should be 66 assertTrue(veryJankyDelta <= 1); 67 } 68 69 public void testJankyDrawFrame() throws Exception { 70 GraphicsStatsProto[] results = runDrawTest("testDrawJankyFrames"); 71 GraphicsStatsProto statsBefore = results[0]; 72 GraphicsStatsProto statsAfter = results[1]; 73 GraphicsStatsJankSummaryProto summaryBefore = statsBefore.getSummary(); 74 GraphicsStatsJankSummaryProto summaryAfter = statsAfter.getSummary(); 75 assertTrue(summaryAfter.getTotalFrames() > summaryBefore.getTotalFrames()); 76 77 int frameDelta = summaryAfter.getTotalFrames() - summaryBefore.getTotalFrames(); 78 int jankyDelta = summaryAfter.getJankyFrames() - summaryBefore.getJankyFrames(); 79 // Test draws 50 frames + 1 initial frame. We expect 40 of them to be janky, 80 // 10 of each of ANIMATION, LAYOUT, RECORD_DRAW, and MISSED_VSYNC 81 assertEquals(51, frameDelta); 82 assertTrue(jankyDelta >= 40); 83 assertTrue(jankyDelta < 45); 84 85 // Although our current stats don't distinguish between ANIMATION, LAYOUT, and RECORD_DRAW 86 // so this will just be slowUi +30 87 int slowUiDelta = summaryAfter.getSlowUiThreadCount() - summaryBefore.getSlowUiThreadCount(); 88 assertTrue(slowUiDelta >= 30); 89 int missedVsyncDelta = summaryAfter.getMissedVsyncCount() 90 - summaryBefore.getMissedVsyncCount(); 91 assertTrue(missedVsyncDelta >= 10); 92 assertTrue(missedVsyncDelta <= 11); 93 94 int veryJankyDelta = countFramesAbove(statsAfter, 60) - countFramesAbove(statsBefore, 60); 95 // The 1st frame could be >40ms, but nothing after that should be 96 assertTrue(veryJankyDelta <= 1); 97 } 98 99 public void testDaveyDrawFrame() throws Exception { 100 GraphicsStatsProto[] results = runDrawTest("testDrawDaveyFrames"); 101 GraphicsStatsProto statsBefore = results[0]; 102 GraphicsStatsProto statsAfter = results[1]; 103 GraphicsStatsJankSummaryProto summaryBefore = statsBefore.getSummary(); 104 GraphicsStatsJankSummaryProto summaryAfter = statsAfter.getSummary(); 105 assertTrue(summaryAfter.getTotalFrames() > summaryBefore.getTotalFrames()); 106 107 int frameDelta = summaryAfter.getTotalFrames() - summaryBefore.getTotalFrames(); 108 int jankyDelta = summaryAfter.getJankyFrames() - summaryBefore.getJankyFrames(); 109 // Test draws 40 frames + 1 initial frame. We expect 10 of them to be daveys, 110 // 10 of them to be daveyjrs, and 20 to jank from missed vsync (from the davey/daveyjr prior to it) 111 assertEquals(41, frameDelta); 112 assertTrue(jankyDelta >= 20); 113 assertTrue(jankyDelta < 25); 114 115 int gt150msDelta = countFramesAbove(statsAfter, 150) - countFramesAbove(statsBefore, 150); 116 assertTrue(gt150msDelta >= 20); // 10 davey jrs + 10 daveys + maybe first frame 117 assertTrue(gt150msDelta <= 21); 118 int gt700msDelta = countFramesAbove(statsAfter, 700) - countFramesAbove(statsBefore, 700); 119 assertEquals(10, gt700msDelta); // 10 daveys 120 } 121 122 private GraphicsStatsProto[] runDrawTest(String testName) throws Exception { 123 return doRunDrawTest(testName, true); 124 } 125 126 private GraphicsStatsProto[] doRunDrawTest(String testName, boolean canRetry) throws Exception { 127 GraphicsStatsProto statsBefore = fetchStats(); 128 assertNotNull(statsBefore); 129 killTestApp(); 130 runDeviceTests(DEVICE_SIDE_TEST_PACKAGE, ".SimpleDrawFrameTests", testName); 131 killTestApp(); 132 GraphicsStatsProto statsAfter = fetchStats(); 133 assertNotNull(statsAfter); 134 // If we get extremely unlucky a log rotate might have happened. If so we retry, but only once 135 // It's a failure if this test takes >24 hours such that 2 rotates could happen while running 136 // this test case, or more likely if stats are not being merged/persisted properly 137 if (canRetry) { 138 if (statsBefore.getStatsStart() != statsAfter.getStatsStart()) { 139 return doRunDrawTest(testName, false); 140 } 141 } else { 142 assertEquals(statsBefore.getStatsStart(), statsAfter.getStatsStart()); 143 } 144 validate(statsBefore); 145 validate(statsAfter); 146 return new GraphicsStatsProto[] { statsBefore, statsAfter }; 147 } 148 149 private void validate(GraphicsStatsProto proto) { 150 assertNotNull(proto.getPackageName()); 151 assertFalse(proto.getPackageName().isEmpty()); 152 assertTrue(proto.getVersionCode() > 0); 153 assertTrue(proto.getStatsStart() > 0); 154 assertTrue(proto.getStatsEnd() > 0); 155 assertTrue(proto.hasSummary()); 156 GraphicsStatsJankSummaryProto summary = proto.getSummary(); 157 assertTrue(summary.getTotalFrames() > 0); 158 // Our test app won't produce that many frames, so we can assert this is a realistic 159 // number. We cap it at 1,000,000 in case the test is repeated many, many times in one day 160 assertTrue(summary.getTotalFrames() < 1000000); 161 // We can't generically assert things about the janky frames, so just assert they fall into 162 // valid ranges. 163 assertTrue(summary.getJankyFrames() <= summary.getTotalFrames()); 164 assertTrue(summary.getMissedVsyncCount() <= summary.getJankyFrames()); 165 assertTrue(summary.getHighInputLatencyCount() <= summary.getJankyFrames()); 166 assertTrue(summary.getSlowUiThreadCount() <= summary.getJankyFrames()); 167 assertTrue(summary.getSlowBitmapUploadCount() <= summary.getJankyFrames()); 168 assertTrue(summary.getSlowDrawCount() <= summary.getJankyFrames()); 169 assertTrue(proto.getHistogramCount() > 0); 170 171 int histogramTotal = countTotalFrames(proto); 172 assertEquals(summary.getTotalFrames(), histogramTotal); 173 } 174 175 private int countFramesAbove(GraphicsStatsProto proto, int thresholdMs) { 176 int totalFrames = 0; 177 for (GraphicsStatsHistogramBucketProto bucket : proto.getHistogramList()) { 178 if (bucket.getRenderMillis() >= thresholdMs) { 179 totalFrames += bucket.getFrameCount(); 180 } 181 } 182 return totalFrames; 183 } 184 185 private int countTotalFrames(GraphicsStatsProto proto) { 186 return countFramesAbove(proto, 0); 187 } 188 189 private void killTestApp() throws Exception { 190 getDevice().executeShellCommand("am kill " + DEVICE_SIDE_TEST_PACKAGE); 191 } 192 193 private GraphicsStatsProto fetchStats() throws Exception { 194 GraphicsStatsServiceDumpProto serviceDumpProto = getDump(GraphicsStatsServiceDumpProto.parser(), 195 "dumpsys graphicsstats --proto"); 196 List<GraphicsStatsProto> protos = filterPackage(serviceDumpProto, DEVICE_SIDE_TEST_PACKAGE); 197 return findLatest(protos); 198 } 199 200 private List<GraphicsStatsProto> filterPackage(GraphicsStatsServiceDumpProto dump, String pkgName) { 201 return filterPackage(dump.getStatsList(), pkgName); 202 } 203 204 private List<GraphicsStatsProto> filterPackage(List<GraphicsStatsProto> list, String pkgName) { 205 ArrayList<GraphicsStatsProto> filtered = new ArrayList<>(); 206 for (GraphicsStatsProto proto : list) { 207 if (pkgName.equals(proto.getPackageName())) { 208 filtered.add(proto); 209 } 210 } 211 return filtered; 212 } 213 214 private GraphicsStatsProto findLatest(List<GraphicsStatsProto> list) { 215 if (list.size() == 0) { return null; } 216 GraphicsStatsProto latest = list.get(0); 217 Date latestDate = new Date(); 218 Date compareTo = new Date(); 219 latestDate.setTime(latest.getStatsEnd()); 220 for (int i = 1; i < list.size(); i++) { 221 GraphicsStatsProto proto = list.get(i); 222 compareTo.setTime(proto.getStatsEnd()); 223 if (compareTo.after(latestDate)) { 224 latestDate.setTime(proto.getStatsEnd()); 225 latest = proto; 226 } 227 } 228 return latest; 229 } 230 } 231
