1 # Copyright 2015 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 7 from telemetry.util import statistics 8 9 DISPLAY_HERTZ = 60.0 10 VSYNC_DURATION = 1e6 / DISPLAY_HERTZ 11 # When to consider a frame frozen (in VSYNC units): meaning 1 initial 12 # frame + 5 repeats of that frame. 13 FROZEN_THRESHOLD = 6 14 # Severity factor. 15 SEVERITY = 3 16 17 IDEAL_RENDER_INSTANT = 'Ideal Render Instant' 18 ACTUAL_RENDER_BEGIN = 'Actual Render Begin' 19 ACTUAL_RENDER_END = 'Actual Render End' 20 SERIAL = 'Serial' 21 22 23 class TimeStats(object): 24 """Stats container for webrtc rendering metrics.""" 25 26 def __init__(self, drift_time=None, mean_drift_time=None, 27 std_dev_drift_time=None, percent_badly_out_of_sync=None, 28 percent_out_of_sync=None, smoothness_score=None, freezing_score=None, 29 rendering_length_error=None, fps=None, frame_distribution=None): 30 self.drift_time = drift_time 31 self.mean_drift_time = mean_drift_time 32 self.std_dev_drift_time = std_dev_drift_time 33 self.percent_badly_out_of_sync = percent_badly_out_of_sync 34 self.percent_out_of_sync = percent_out_of_sync 35 self.smoothness_score = smoothness_score 36 self.freezing_score = freezing_score 37 self.rendering_length_error = rendering_length_error 38 self.fps = fps 39 self.frame_distribution = frame_distribution 40 self.invalid_data = False 41 42 43 44 class WebMediaPlayerMsRenderingStats(object): 45 """Analyzes events of WebMediaPlayerMs type.""" 46 47 def __init__(self, events): 48 """Save relevant events according to their stream.""" 49 self.stream_to_events = self._MapEventsToStream(events) 50 51 def _IsEventValid(self, event): 52 """Check that the needed arguments are present in event. 53 54 Args: 55 event: event to check. 56 57 Returns: 58 True is event is valid, false otherwise.""" 59 if not event.args: 60 return False 61 mandatory = [ACTUAL_RENDER_BEGIN, ACTUAL_RENDER_END, 62 IDEAL_RENDER_INSTANT, SERIAL] 63 for parameter in mandatory: 64 if not parameter in event.args: 65 return False 66 return True 67 68 def _MapEventsToStream(self, events): 69 """Build a dictionary of events indexed by stream. 70 71 The events of interest have a 'Serial' argument which represents the 72 stream ID. The 'Serial' argument identifies the local or remote nature of 73 the stream with a least significant bit of 0 or 1 as well as the hash 74 value of the video track's URL. So stream::=hash(0|1} . The method will 75 then list the events of the same stream in a frame_distribution on stream 76 id. Practically speaking remote streams have an odd stream id and local 77 streams have a even stream id. 78 Args: 79 events: Telemetry WebMediaPlayerMs events. 80 81 Returns: 82 A dict of stream IDs mapped to events on that stream. 83 """ 84 stream_to_events = {} 85 for event in events: 86 if not self._IsEventValid(event): 87 # This is not a render event, skip it. 88 continue 89 stream = event.args[SERIAL] 90 events_for_stream = stream_to_events.setdefault(stream, []) 91 events_for_stream.append(event) 92 93 return stream_to_events 94 95 def _GetCadence(self, relevant_events): 96 """Calculate the apparent cadence of the rendering. 97 98 In this paragraph I will be using regex notation. What is intended by the 99 word cadence is a sort of extended instantaneous 'Cadence' (thus not 100 necessarily periodic). Just as an example, a normal 'Cadence' could be 101 something like [2 3] which means possibly an observed frame persistence 102 progression of [{2 3}+] for an ideal 20FPS video source. So what we are 103 calculating here is the list of frame persistence, kind of a 104 'Proto-Cadence', but cadence is shorter so we abuse the word. 105 106 Args: 107 relevant_events: list of Telemetry events. 108 109 Returns: 110 a list of frame persistence values. 111 """ 112 cadence = [] 113 frame_persistence = 0 114 old_ideal_render = 0 115 for event in relevant_events: 116 if not self._IsEventValid(event): 117 # This event is not a render event so skip it. 118 continue 119 if event.args[IDEAL_RENDER_INSTANT] == old_ideal_render: 120 frame_persistence += 1 121 else: 122 cadence.append(frame_persistence) 123 frame_persistence = 1 124 old_ideal_render = event.args[IDEAL_RENDER_INSTANT] 125 cadence.append(frame_persistence) 126 cadence.pop(0) 127 return cadence 128 129 def _GetSourceToOutputDistribution(self, cadence): 130 """Create distribution for the cadence frame display values. 131 132 If the overall display distribution is A1:A2:..:An, this will tell us how 133 many times a frame stays displayed during Ak*VSYNC_DURATION, also known as 134 'source to output' distribution. Or in other terms: 135 a distribution B::= let C be the cadence, B[k]=p with k in Unique(C) 136 and p=Card(k in C). 137 138 Args: 139 cadence: list of frame persistence values. 140 141 Returns: 142 a dictionary containing the distribution 143 """ 144 frame_distribution = {} 145 for ticks in cadence: 146 ticks_so_far = frame_distribution.setdefault(ticks, 0) 147 frame_distribution[ticks] = ticks_so_far + 1 148 return frame_distribution 149 150 def _GetFpsFromCadence(self, frame_distribution): 151 """Calculate the apparent FPS from frame distribution. 152 153 Knowing the display frequency and the frame distribution, it is possible to 154 calculate the video apparent frame rate as played by WebMediaPlayerMs 155 module. 156 157 Args: 158 frame_distribution: the source to output distribution. 159 160 Returns: 161 the video apparent frame rate. 162 """ 163 number_frames = sum(frame_distribution.values()) 164 number_vsyncs = sum([ticks * frame_distribution[ticks] 165 for ticks in frame_distribution]) 166 mean_ratio = float(number_vsyncs) / number_frames 167 return DISPLAY_HERTZ / mean_ratio 168 169 def _GetFrozenFramesReports(self, frame_distribution): 170 """Find evidence of frozen frames in distribution. 171 172 For simplicity we count as freezing the frames that appear at least five 173 times in a row counted from 'Ideal Render Instant' perspective. So let's 174 say for 1 source frame, we rendered 6 frames, then we consider 5 of these 175 rendered frames as frozen. But we mitigate this by saying anything under 176 5 frozen frames will not be counted as frozen. 177 178 Args: 179 frame_distribution: the source to output distribution. 180 181 Returns: 182 a list of dicts whose keys are ('frozen_frames', 'occurrences'). 183 """ 184 frozen_frames = [] 185 frozen_frame_vsyncs = [ticks for ticks in frame_distribution if ticks >= 186 FROZEN_THRESHOLD] 187 for frozen_frames_vsync in frozen_frame_vsyncs: 188 logging.debug('%s frames not updated after %s vsyncs', 189 frame_distribution[frozen_frames_vsync], frozen_frames_vsync) 190 frozen_frames.append( 191 {'frozen_frames': frozen_frames_vsync - 1, 192 'occurrences': frame_distribution[frozen_frames_vsync]}) 193 return frozen_frames 194 195 def _FrozenPenaltyWeight(self, number_frozen_frames): 196 """Returns the weighted penalty for a number of frozen frames. 197 198 As mentioned earlier, we count for frozen anything above 6 vsync display 199 duration for the same 'Initial Render Instant', which is five frozen 200 frames. 201 202 Args: 203 number_frozen_frames: number of frozen frames. 204 205 Returns: 206 the penalty weight (int) for that number of frozen frames. 207 """ 208 209 penalty = { 210 0: 0, 211 1: 0, 212 2: 0, 213 3: 0, 214 4: 0, 215 5: 1, 216 6: 5, 217 7: 15, 218 8: 25 219 } 220 weight = penalty.get(number_frozen_frames, 8 * (number_frozen_frames - 4)) 221 return weight 222 223 def _IsRemoteStream(self, stream): 224 """Check if stream is remote.""" 225 return stream % 2 226 227 def _GetDrifTimeStats(self, relevant_events, cadence): 228 """Get the drift time statistics. 229 230 This method will calculate drift_time stats, that is to say : 231 drift_time::= list(actual render begin - ideal render). 232 rendering_length error::= the rendering length error. 233 234 Args: 235 relevant_events: events to get drift times stats from. 236 cadence: list of frame persistence values. 237 238 Returns: 239 a tuple of (drift_time, rendering_length_error). 240 """ 241 drift_time = [] 242 old_ideal_render = 0 243 discrepancy = [] 244 index = 0 245 for event in relevant_events: 246 current_ideal_render = event.args[IDEAL_RENDER_INSTANT] 247 if current_ideal_render == old_ideal_render: 248 # Skip to next event because we're looking for a source frame. 249 continue 250 actual_render_begin = event.args[ACTUAL_RENDER_BEGIN] 251 drift_time.append(actual_render_begin - current_ideal_render) 252 discrepancy.append(abs(current_ideal_render - old_ideal_render 253 - VSYNC_DURATION * cadence[index])) 254 old_ideal_render = current_ideal_render 255 index += 1 256 discrepancy.pop(0) 257 last_ideal_render = relevant_events[-1].args[IDEAL_RENDER_INSTANT] 258 first_ideal_render = relevant_events[0].args[IDEAL_RENDER_INSTANT] 259 rendering_length_error = 100.0 * (sum([x for x in discrepancy]) / 260 (last_ideal_render - first_ideal_render)) 261 262 return drift_time, rendering_length_error 263 264 def _GetSmoothnessStats(self, norm_drift_time): 265 """Get the smoothness stats from the normalized drift time. 266 267 This method will calculate the smoothness score, along with the percentage 268 of frames badly out of sync and the percentage of frames out of sync. To be 269 considered badly out of sync, a frame has to have missed rendering by at 270 least 2*VSYNC_DURATION. To be considered out of sync, a frame has to have 271 missed rendering by at least one VSYNC_DURATION. 272 The smoothness score is a measure of how out of sync the frames are. 273 274 Args: 275 norm_drift_time: normalized drift time. 276 277 Returns: 278 a tuple of (percent_badly_oos, percent_out_of_sync, smoothness_score) 279 """ 280 # How many times is a frame later/earlier than T=2*VSYNC_DURATION. Time is 281 # in microseconds. 282 frames_severely_out_of_sync = len( 283 [x for x in norm_drift_time if abs(x) > 2 * VSYNC_DURATION]) 284 percent_badly_oos = ( 285 100.0 * frames_severely_out_of_sync / len(norm_drift_time)) 286 287 # How many times is a frame later/earlier than VSYNC_DURATION. 288 frames_out_of_sync = len( 289 [x for x in norm_drift_time if abs(x) > VSYNC_DURATION]) 290 percent_out_of_sync = ( 291 100.0 * frames_out_of_sync / len(norm_drift_time)) 292 293 frames_oos_only_once = frames_out_of_sync - frames_severely_out_of_sync 294 295 # Calculate smoothness metric. From the formula, we can see that smoothness 296 # score can be negative. 297 smoothness_score = 100.0 - 100.0 * (frames_oos_only_once + 298 SEVERITY * frames_severely_out_of_sync) / len(norm_drift_time) 299 300 # Minimum smoothness_score value allowed is zero. 301 if smoothness_score < 0: 302 smoothness_score = 0 303 304 return (percent_badly_oos, percent_out_of_sync, smoothness_score) 305 306 def _GetFreezingScore(self, frame_distribution): 307 """Get the freezing score.""" 308 309 # The freezing score is based on the source to output distribution. 310 number_vsyncs = sum([n * frame_distribution[n] 311 for n in frame_distribution]) 312 frozen_frames = self._GetFrozenFramesReports(frame_distribution) 313 314 # Calculate freezing metric. 315 # Freezing metric can be negative if things are really bad. In that case we 316 # change it to zero as minimum valud. 317 freezing_score = 100.0 318 for frozen_report in frozen_frames: 319 weight = self._FrozenPenaltyWeight(frozen_report['frozen_frames']) 320 freezing_score -= ( 321 100.0 * frozen_report['occurrences'] / number_vsyncs * weight) 322 if freezing_score < 0: 323 freezing_score = 0 324 325 return freezing_score 326 327 def GetTimeStats(self): 328 """Calculate time stamp stats for all remote stream events.""" 329 stats = {} 330 for stream, relevant_events in self.stream_to_events.iteritems(): 331 if len(relevant_events) == 1: 332 logging.debug('Found a stream=%s with just one event', stream) 333 continue 334 if not self._IsRemoteStream(stream): 335 logging.info('Skipping processing of local stream: %s', stream) 336 continue 337 338 cadence = self._GetCadence(relevant_events) 339 if not cadence: 340 stats = TimeStats() 341 stats.invalid_data = True 342 return stats 343 344 frame_distribution = self._GetSourceToOutputDistribution(cadence) 345 fps = self._GetFpsFromCadence(frame_distribution) 346 347 drift_time_stats = self._GetDrifTimeStats(relevant_events, cadence) 348 (drift_time, rendering_length_error) = drift_time_stats 349 350 # Drift time normalization. 351 mean_drift_time = statistics.ArithmeticMean(drift_time) 352 norm_drift_time = [abs(x - mean_drift_time) for x in drift_time] 353 354 smoothness_stats = self._GetSmoothnessStats(norm_drift_time) 355 (percent_badly_oos, percent_out_of_sync, 356 smoothness_score) = smoothness_stats 357 358 freezing_score = self._GetFreezingScore(frame_distribution) 359 360 stats = TimeStats(drift_time=drift_time, 361 percent_badly_out_of_sync=percent_badly_oos, 362 percent_out_of_sync=percent_out_of_sync, 363 smoothness_score=smoothness_score, freezing_score=freezing_score, 364 rendering_length_error=rendering_length_error, fps=fps, 365 frame_distribution=frame_distribution) 366 return stats 367
