1 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 7 from telemetry.util import perf_tests_helper 8 from telemetry.util import statistics 9 from telemetry.value import improvement_direction 10 from telemetry.value import list_of_scalar_values 11 from telemetry.value import scalar 12 from telemetry.web_perf.metrics import rendering_stats 13 from telemetry.web_perf.metrics import timeline_based_metric 14 15 16 NOT_ENOUGH_FRAMES_MESSAGE = ( 17 'Not enough frames for smoothness metrics (at least two are required).\n' 18 'Issues that have caused this in the past:\n' 19 '- Browser bugs that prevents the page from redrawing\n' 20 '- Bugs in the synthetic gesture code\n' 21 '- Page and benchmark out of sync (e.g. clicked element was renamed)\n' 22 '- Pages that render extremely slow\n' 23 '- Pages that can\'t be scrolled') 24 25 26 class SmoothnessMetric(timeline_based_metric.TimelineBasedMetric): 27 """Computes metrics that measure smoothness of animations over given ranges. 28 29 Animations are typically considered smooth if the frame rates are close to 30 60 frames per second (fps) and uniformly distributed over the sequence. To 31 determine if a timeline range contains a smooth animation, we update the 32 results object with several representative metrics: 33 34 frame_times: A list of raw frame times 35 mean_frame_time: The arithmetic mean of frame times 36 percentage_smooth: Percentage of frames that were hitting 60 FPS. 37 frame_time_discrepancy: The absolute discrepancy of frame timestamps 38 mean_pixels_approximated: The mean percentage of pixels approximated 39 queueing_durations: The queueing delay between compositor & main threads 40 41 Note that if any of the interaction records provided to AddResults have less 42 than 2 frames, we will return telemetry values with None values for each of 43 the smoothness metrics. Similarly, older browsers without support for 44 tracking the BeginMainFrame events will report a ListOfScalarValues with a 45 None value for the queueing duration metric. 46 """ 47 48 def __init__(self): 49 super(SmoothnessMetric, self).__init__() 50 51 def AddResults(self, model, renderer_thread, interaction_records, results): 52 self.VerifyNonOverlappedRecords(interaction_records) 53 renderer_process = renderer_thread.parent 54 stats = rendering_stats.RenderingStats( 55 renderer_process, model.browser_process, model.surface_flinger_process, 56 model.gpu_process, [r.GetBounds() for r in interaction_records]) 57 has_surface_flinger_stats = model.surface_flinger_process is not None 58 self._PopulateResultsFromStats(results, stats, has_surface_flinger_stats) 59 60 def _PopulateResultsFromStats(self, results, stats, 61 has_surface_flinger_stats): 62 page = results.current_page 63 values = [ 64 self._ComputeQueueingDuration(page, stats), 65 self._ComputeFrameTimeDiscrepancy(page, stats), 66 self._ComputeMeanPixelsApproximated(page, stats), 67 self._ComputeMeanPixelsCheckerboarded(page, stats) 68 ] 69 values += self._ComputeLatencyMetric(page, stats, 'input_event_latency', 70 stats.input_event_latency) 71 values += self._ComputeLatencyMetric(page, stats, 72 'main_thread_scroll_latency', 73 stats.main_thread_scroll_latency) 74 values.append(self._ComputeFirstGestureScrollUpdateLatencies(page, stats)) 75 values += self._ComputeFrameTimeMetric(page, stats) 76 if has_surface_flinger_stats: 77 values += self._ComputeSurfaceFlingerMetric(page, stats) 78 79 for v in values: 80 results.AddValue(v) 81 82 def _HasEnoughFrames(self, list_of_frame_timestamp_lists): 83 """Whether we have collected at least two frames in every timestamp list.""" 84 return all(len(s) >= 2 for s in list_of_frame_timestamp_lists) 85 86 @staticmethod 87 def _GetNormalizedDeltas(data, refresh_period, min_normalized_delta=None): 88 deltas = [t2 - t1 for t1, t2 in zip(data, data[1:])] 89 if min_normalized_delta != None: 90 deltas = [d for d in deltas 91 if d / refresh_period >= min_normalized_delta] 92 return (deltas, [delta / refresh_period for delta in deltas]) 93 94 @staticmethod 95 def _JoinTimestampRanges(frame_timestamps): 96 """Joins ranges of timestamps, adjusting timestamps to remove deltas 97 between the start of a range and the end of the prior range. 98 """ 99 timestamps = [] 100 for timestamp_range in frame_timestamps: 101 if len(timestamps) == 0: 102 timestamps.extend(timestamp_range) 103 else: 104 for i in range(1, len(timestamp_range)): 105 timestamps.append(timestamps[-1] + 106 timestamp_range[i] - timestamp_range[i-1]) 107 return timestamps 108 109 def _ComputeSurfaceFlingerMetric(self, page, stats): 110 jank_count = None 111 avg_surface_fps = None 112 max_frame_delay = None 113 frame_lengths = None 114 none_value_reason = None 115 if self._HasEnoughFrames(stats.frame_timestamps): 116 timestamps = self._JoinTimestampRanges(stats.frame_timestamps) 117 frame_count = len(timestamps) 118 milliseconds = timestamps[-1] - timestamps[0] 119 min_normalized_frame_length = 0.5 120 121 frame_lengths, normalized_frame_lengths = \ 122 self._GetNormalizedDeltas(timestamps, stats.refresh_period, 123 min_normalized_frame_length) 124 if len(frame_lengths) < frame_count - 1: 125 logging.warning('Skipping frame lengths that are too short.') 126 frame_count = len(frame_lengths) + 1 127 if len(frame_lengths) == 0: 128 raise Exception('No valid frames lengths found.') 129 _, normalized_changes = \ 130 self._GetNormalizedDeltas(frame_lengths, stats.refresh_period) 131 jankiness = [max(0, round(change)) for change in normalized_changes] 132 pause_threshold = 20 133 jank_count = sum(1 for change in jankiness 134 if change > 0 and change < pause_threshold) 135 avg_surface_fps = int(round((frame_count - 1) * 1000.0 / milliseconds)) 136 max_frame_delay = round(max(normalized_frame_lengths)) 137 frame_lengths = normalized_frame_lengths 138 else: 139 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE 140 141 return ( 142 scalar.ScalarValue( 143 page, 'avg_surface_fps', 'fps', avg_surface_fps, 144 description='Average frames per second as measured by the ' 145 'platform\'s SurfaceFlinger.', 146 none_value_reason=none_value_reason, 147 improvement_direction=improvement_direction.UP), 148 scalar.ScalarValue( 149 page, 'jank_count', 'janks', jank_count, 150 description='Number of changes in frame rate as measured by the ' 151 'platform\'s SurfaceFlinger.', 152 none_value_reason=none_value_reason, 153 improvement_direction=improvement_direction.DOWN), 154 scalar.ScalarValue( 155 page, 'max_frame_delay', 'vsyncs', max_frame_delay, 156 description='Largest frame time as measured by the platform\'s ' 157 'SurfaceFlinger.', 158 none_value_reason=none_value_reason, 159 improvement_direction=improvement_direction.DOWN), 160 list_of_scalar_values.ListOfScalarValues( 161 page, 'frame_lengths', 'vsyncs', frame_lengths, 162 description='Frame time in vsyncs as measured by the platform\'s ' 163 'SurfaceFlinger.', 164 none_value_reason=none_value_reason, 165 improvement_direction=improvement_direction.DOWN) 166 ) 167 168 def _ComputeLatencyMetric(self, page, stats, name, list_of_latency_lists): 169 """Returns Values for the mean and discrepancy for given latency stats.""" 170 mean_latency = None 171 latency_discrepancy = None 172 none_value_reason = None 173 if self._HasEnoughFrames(stats.frame_timestamps): 174 latency_list = perf_tests_helper.FlattenList(list_of_latency_lists) 175 if len(latency_list) == 0: 176 return () 177 mean_latency = round(statistics.ArithmeticMean(latency_list), 3) 178 latency_discrepancy = ( 179 round(statistics.DurationsDiscrepancy(latency_list), 4)) 180 else: 181 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE 182 return ( 183 scalar.ScalarValue( 184 page, 'mean_%s' % name, 'ms', mean_latency, 185 description='Arithmetic mean of the raw %s values' % name, 186 none_value_reason=none_value_reason, 187 improvement_direction=improvement_direction.DOWN), 188 scalar.ScalarValue( 189 page, '%s_discrepancy' % name, 'ms', latency_discrepancy, 190 description='Discrepancy of the raw %s values' % name, 191 none_value_reason=none_value_reason, 192 improvement_direction=improvement_direction.DOWN) 193 ) 194 195 def _ComputeFirstGestureScrollUpdateLatencies(self, page, stats): 196 """Returns a ListOfScalarValuesValues of gesture scroll update latencies. 197 198 Returns a Value for the first gesture scroll update latency for each 199 interaction record in |stats|. 200 """ 201 none_value_reason = None 202 first_gesture_scroll_update_latencies = [round(latencies[0], 4) 203 for latencies in stats.gesture_scroll_update_latency 204 if len(latencies)] 205 if (not self._HasEnoughFrames(stats.frame_timestamps) or 206 not first_gesture_scroll_update_latencies): 207 first_gesture_scroll_update_latencies = None 208 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE 209 return list_of_scalar_values.ListOfScalarValues( 210 page, 'first_gesture_scroll_update_latency', 'ms', 211 first_gesture_scroll_update_latencies, 212 description='First gesture scroll update latency measures the time it ' 213 'takes to process the very first gesture scroll update ' 214 'input event. The first scroll gesture can often get ' 215 'delayed by work related to page loading.', 216 none_value_reason=none_value_reason, 217 improvement_direction=improvement_direction.DOWN) 218 219 def _ComputeQueueingDuration(self, page, stats): 220 """Returns a Value for the frame queueing durations.""" 221 queueing_durations = None 222 none_value_reason = None 223 if 'frame_queueing_durations' in stats.errors: 224 none_value_reason = stats.errors['frame_queueing_durations'] 225 elif self._HasEnoughFrames(stats.frame_timestamps): 226 queueing_durations = perf_tests_helper.FlattenList( 227 stats.frame_queueing_durations) 228 if len(queueing_durations) == 0: 229 queueing_durations = None 230 none_value_reason = 'No frame queueing durations recorded.' 231 else: 232 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE 233 return list_of_scalar_values.ListOfScalarValues( 234 page, 'queueing_durations', 'ms', queueing_durations, 235 description='The frame queueing duration quantifies how out of sync ' 236 'the compositor and renderer threads are. It is the amount ' 237 'of wall time that elapses between a ' 238 'ScheduledActionSendBeginMainFrame event in the compositor ' 239 'thread and the corresponding BeginMainFrame event in the ' 240 'main thread.', 241 none_value_reason=none_value_reason, 242 improvement_direction=improvement_direction.DOWN) 243 244 def _ComputeFrameTimeMetric(self, page, stats): 245 """Returns Values for the frame time metrics. 246 247 This includes the raw and mean frame times, as well as the percentage of 248 frames that were hitting 60 fps. 249 """ 250 frame_times = None 251 mean_frame_time = None 252 percentage_smooth = None 253 none_value_reason = None 254 if self._HasEnoughFrames(stats.frame_timestamps): 255 frame_times = perf_tests_helper.FlattenList(stats.frame_times) 256 mean_frame_time = round(statistics.ArithmeticMean(frame_times), 3) 257 # We use 17ms as a somewhat looser threshold, instead of 1000.0/60.0. 258 smooth_threshold = 17.0 259 smooth_count = sum(1 for t in frame_times if t < smooth_threshold) 260 percentage_smooth = float(smooth_count) / len(frame_times) * 100.0 261 else: 262 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE 263 return ( 264 list_of_scalar_values.ListOfScalarValues( 265 page, 'frame_times', 'ms', frame_times, 266 description='List of raw frame times, helpful to understand the ' 267 'other metrics.', 268 none_value_reason=none_value_reason, 269 improvement_direction=improvement_direction.DOWN), 270 scalar.ScalarValue( 271 page, 'mean_frame_time', 'ms', mean_frame_time, 272 description='Arithmetic mean of frame times.', 273 none_value_reason=none_value_reason, 274 improvement_direction=improvement_direction.DOWN), 275 scalar.ScalarValue( 276 page, 'percentage_smooth', 'score', percentage_smooth, 277 description='Percentage of frames that were hitting 60 fps.', 278 none_value_reason=none_value_reason, 279 improvement_direction=improvement_direction.UP) 280 ) 281 282 def _ComputeFrameTimeDiscrepancy(self, page, stats): 283 """Returns a Value for the absolute discrepancy of frame time stamps.""" 284 285 frame_discrepancy = None 286 none_value_reason = None 287 if self._HasEnoughFrames(stats.frame_timestamps): 288 frame_discrepancy = round(statistics.TimestampsDiscrepancy( 289 stats.frame_timestamps), 4) 290 else: 291 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE 292 return scalar.ScalarValue( 293 page, 'frame_time_discrepancy', 'ms', frame_discrepancy, 294 description='Absolute discrepancy of frame time stamps, where ' 295 'discrepancy is a measure of irregularity. It quantifies ' 296 'the worst jank. For a single pause, discrepancy ' 297 'corresponds to the length of this pause in milliseconds. ' 298 'Consecutive pauses increase the discrepancy. This metric ' 299 'is important because even if the mean and 95th ' 300 'percentile are good, one long pause in the middle of an ' 301 'interaction is still bad.', 302 none_value_reason=none_value_reason, 303 improvement_direction=improvement_direction.DOWN) 304 305 def _ComputeMeanPixelsApproximated(self, page, stats): 306 """Add the mean percentage of pixels approximated. 307 308 This looks at tiles which are missing or of low or non-ideal resolution. 309 """ 310 mean_pixels_approximated = None 311 none_value_reason = None 312 if self._HasEnoughFrames(stats.frame_timestamps): 313 mean_pixels_approximated = round(statistics.ArithmeticMean( 314 perf_tests_helper.FlattenList( 315 stats.approximated_pixel_percentages)), 3) 316 else: 317 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE 318 return scalar.ScalarValue( 319 page, 'mean_pixels_approximated', 'percent', mean_pixels_approximated, 320 description='Percentage of pixels that were approximated ' 321 '(checkerboarding, low-resolution tiles, etc.).', 322 none_value_reason=none_value_reason, 323 improvement_direction=improvement_direction.DOWN) 324 325 def _ComputeMeanPixelsCheckerboarded(self, page, stats): 326 """Add the mean percentage of pixels checkerboarded. 327 328 This looks at tiles which are only missing. 329 It does not take into consideration tiles which are of low or 330 non-ideal resolution. 331 """ 332 mean_pixels_checkerboarded = None 333 none_value_reason = None 334 if self._HasEnoughFrames(stats.frame_timestamps): 335 if rendering_stats.CHECKERBOARDED_PIXEL_ERROR in stats.errors: 336 none_value_reason = stats.errors[ 337 rendering_stats.CHECKERBOARDED_PIXEL_ERROR] 338 else: 339 mean_pixels_checkerboarded = round(statistics.ArithmeticMean( 340 perf_tests_helper.FlattenList( 341 stats.checkerboarded_pixel_percentages)), 3) 342 else: 343 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE 344 return scalar.ScalarValue( 345 page, 'mean_pixels_checkerboarded', 'percent', 346 mean_pixels_checkerboarded, 347 description='Percentage of pixels that were checkerboarded.', 348 none_value_reason=none_value_reason, 349 improvement_direction=improvement_direction.DOWN) 350
