1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "media/cast/video_receiver/video_receiver.h" 6 7 #include <algorithm> 8 9 #include "base/bind.h" 10 #include "base/logging.h" 11 #include "base/message_loop/message_loop.h" 12 #include "crypto/encryptor.h" 13 #include "crypto/symmetric_key.h" 14 #include "media/cast/cast_defines.h" 15 #include "media/cast/framer/framer.h" 16 #include "media/cast/video_receiver/video_decoder.h" 17 18 namespace media { 19 namespace cast { 20 21 const int64 kMinSchedulingDelayMs = 1; 22 23 static const int64 kMinTimeBetweenOffsetUpdatesMs = 2000; 24 static const int kTimeOffsetFilter = 8; 25 static const int64_t kMinProcessIntervalMs = 5; 26 27 // Local implementation of RtpData (defined in rtp_rtcp_defines.h). 28 // Used to pass payload data into the video receiver. 29 class LocalRtpVideoData : public RtpData { 30 public: 31 explicit LocalRtpVideoData(VideoReceiver* video_receiver) 32 : video_receiver_(video_receiver) {} 33 34 virtual ~LocalRtpVideoData() {} 35 36 virtual void OnReceivedPayloadData(const uint8* payload_data, 37 size_t payload_size, 38 const RtpCastHeader* rtp_header) OVERRIDE { 39 video_receiver_->IncomingParsedRtpPacket(payload_data, payload_size, 40 *rtp_header); 41 } 42 43 private: 44 VideoReceiver* video_receiver_; 45 }; 46 47 // Local implementation of RtpPayloadFeedback (defined in rtp_defines.h) 48 // Used to convey cast-specific feedback from receiver to sender. 49 // Callback triggered by the Framer (cast message builder). 50 class LocalRtpVideoFeedback : public RtpPayloadFeedback { 51 public: 52 explicit LocalRtpVideoFeedback(VideoReceiver* video_receiver) 53 : video_receiver_(video_receiver) { 54 } 55 56 virtual void CastFeedback(const RtcpCastMessage& cast_message) OVERRIDE { 57 video_receiver_->CastFeedback(cast_message); 58 } 59 60 private: 61 VideoReceiver* video_receiver_; 62 }; 63 64 // Local implementation of RtpReceiverStatistics (defined by rtcp.h). 65 // Used to pass statistics data from the RTP module to the RTCP module. 66 class LocalRtpReceiverStatistics : public RtpReceiverStatistics { 67 public: 68 explicit LocalRtpReceiverStatistics(RtpReceiver* rtp_receiver) 69 : rtp_receiver_(rtp_receiver) { 70 } 71 72 virtual void GetStatistics(uint8* fraction_lost, 73 uint32* cumulative_lost, // 24 bits valid. 74 uint32* extended_high_sequence_number, 75 uint32* jitter) OVERRIDE { 76 rtp_receiver_->GetStatistics(fraction_lost, 77 cumulative_lost, 78 extended_high_sequence_number, 79 jitter); 80 } 81 82 private: 83 RtpReceiver* rtp_receiver_; 84 }; 85 86 VideoReceiver::VideoReceiver(scoped_refptr<CastEnvironment> cast_environment, 87 const VideoReceiverConfig& video_config, 88 PacedPacketSender* const packet_sender) 89 : cast_environment_(cast_environment), 90 codec_(video_config.codec), 91 target_delay_delta_( 92 base::TimeDelta::FromMilliseconds(video_config.rtp_max_delay_ms)), 93 frame_delay_(base::TimeDelta::FromMilliseconds( 94 1000 / video_config.max_frame_rate)), 95 incoming_payload_callback_(new LocalRtpVideoData(this)), 96 incoming_payload_feedback_(new LocalRtpVideoFeedback(this)), 97 rtp_receiver_(cast_environment_->Clock(), NULL, &video_config, 98 incoming_payload_callback_.get()), 99 rtp_video_receiver_statistics_( 100 new LocalRtpReceiverStatistics(&rtp_receiver_)), 101 time_incoming_packet_updated_(false), 102 incoming_rtp_timestamp_(0), 103 weak_factory_(this) { 104 int max_unacked_frames = video_config.rtp_max_delay_ms * 105 video_config.max_frame_rate / 1000; 106 DCHECK(max_unacked_frames) << "Invalid argument"; 107 108 if (video_config.aes_iv_mask.size() == kAesKeySize && 109 video_config.aes_key.size() == kAesKeySize) { 110 iv_mask_ = video_config.aes_iv_mask; 111 crypto::SymmetricKey* key = crypto::SymmetricKey::Import( 112 crypto::SymmetricKey::AES, video_config.aes_key); 113 decryptor_.reset(new crypto::Encryptor()); 114 decryptor_->Init(key, crypto::Encryptor::CTR, std::string()); 115 } else if (video_config.aes_iv_mask.size() != 0 || 116 video_config.aes_key.size() != 0) { 117 DCHECK(false) << "Invalid crypto configuration"; 118 } 119 120 framer_.reset(new Framer(cast_environment->Clock(), 121 incoming_payload_feedback_.get(), 122 video_config.incoming_ssrc, 123 video_config.decoder_faster_than_max_frame_rate, 124 max_unacked_frames)); 125 if (!video_config.use_external_decoder) { 126 video_decoder_.reset(new VideoDecoder(video_config, cast_environment)); 127 } 128 129 rtcp_.reset( 130 new Rtcp(cast_environment_, 131 NULL, 132 packet_sender, 133 NULL, 134 rtp_video_receiver_statistics_.get(), 135 video_config.rtcp_mode, 136 base::TimeDelta::FromMilliseconds(video_config.rtcp_interval), 137 video_config.feedback_ssrc, 138 video_config.incoming_ssrc, 139 video_config.rtcp_c_name)); 140 } 141 142 VideoReceiver::~VideoReceiver() {} 143 144 void VideoReceiver::InitializeTimers() { 145 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 146 ScheduleNextRtcpReport(); 147 ScheduleNextCastMessage(); 148 } 149 150 void VideoReceiver::GetRawVideoFrame( 151 const VideoFrameDecodedCallback& callback) { 152 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 153 GetEncodedVideoFrame(base::Bind(&VideoReceiver::DecodeVideoFrame, 154 base::Unretained(this), callback)); 155 } 156 157 // Called when we have a frame to decode. 158 void VideoReceiver::DecodeVideoFrame( 159 const VideoFrameDecodedCallback& callback, 160 scoped_ptr<EncodedVideoFrame> encoded_frame, 161 const base::TimeTicks& render_time) { 162 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 163 // Hand the ownership of the encoded frame to the decode thread. 164 cast_environment_->PostTask(CastEnvironment::VIDEO_DECODER, FROM_HERE, 165 base::Bind(&VideoReceiver::DecodeVideoFrameThread, base::Unretained(this), 166 base::Passed(&encoded_frame), render_time, callback)); 167 } 168 169 // Utility function to run the decoder on a designated decoding thread. 170 void VideoReceiver::DecodeVideoFrameThread( 171 scoped_ptr<EncodedVideoFrame> encoded_frame, 172 const base::TimeTicks render_time, 173 const VideoFrameDecodedCallback& frame_decoded_callback) { 174 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::VIDEO_DECODER)); 175 DCHECK(video_decoder_); 176 177 if (!(video_decoder_->DecodeVideoFrame(encoded_frame.get(), render_time, 178 frame_decoded_callback))) { 179 // This will happen if we decide to decode but not show a frame. 180 cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, 181 base::Bind(&VideoReceiver::GetRawVideoFrame, base::Unretained(this), 182 frame_decoded_callback)); 183 } 184 } 185 186 bool VideoReceiver::DecryptVideoFrame( 187 scoped_ptr<EncodedVideoFrame>* video_frame) { 188 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 189 DCHECK(decryptor_) << "Invalid state"; 190 191 if (!decryptor_->SetCounter(GetAesNonce((*video_frame)->frame_id, 192 iv_mask_))) { 193 NOTREACHED() << "Failed to set counter"; 194 return false; 195 } 196 std::string decrypted_video_data; 197 if (!decryptor_->Decrypt((*video_frame)->data, &decrypted_video_data)) { 198 VLOG(1) << "Decryption error"; 199 // Give up on this frame, release it from jitter buffer. 200 framer_->ReleaseFrame((*video_frame)->frame_id); 201 return false; 202 } 203 (*video_frame)->data.swap(decrypted_video_data); 204 return true; 205 } 206 207 // Called from the main cast thread. 208 void VideoReceiver::GetEncodedVideoFrame( 209 const VideoFrameEncodedCallback& callback) { 210 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 211 scoped_ptr<EncodedVideoFrame> encoded_frame(new EncodedVideoFrame()); 212 uint32 rtp_timestamp = 0; 213 bool next_frame = false; 214 215 if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &rtp_timestamp, 216 &next_frame)) { 217 // We have no video frames. Wait for new packet(s). 218 queued_encoded_callbacks_.push_back(callback); 219 return; 220 } 221 222 if (decryptor_ && !DecryptVideoFrame(&encoded_frame)) { 223 // Logging already done. 224 queued_encoded_callbacks_.push_back(callback); 225 return; 226 } 227 228 base::TimeTicks render_time; 229 if (PullEncodedVideoFrame(rtp_timestamp, next_frame, &encoded_frame, 230 &render_time)) { 231 cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, 232 base::Bind(callback, base::Passed(&encoded_frame), render_time)); 233 } else { 234 // We have a video frame; however we are missing packets and we have time 235 // to wait for new packet(s). 236 queued_encoded_callbacks_.push_back(callback); 237 } 238 } 239 240 // Should we pull the encoded video frame from the framer? decided by if this is 241 // the next frame or we are running out of time and have to pull the following 242 // frame. 243 // If the frame is too old to be rendered we set the don't show flag in the 244 // video bitstream where possible. 245 bool VideoReceiver::PullEncodedVideoFrame(uint32 rtp_timestamp, 246 bool next_frame, scoped_ptr<EncodedVideoFrame>* encoded_frame, 247 base::TimeTicks* render_time) { 248 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 249 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); 250 *render_time = GetRenderTime(now, rtp_timestamp); 251 252 // TODO(mikhal): Store actual render time and not diff. 253 cast_environment_->Logging()->InsertFrameEventWithDelay(kVideoRenderDelay, 254 rtp_timestamp, (*encoded_frame)->frame_id, now - *render_time); 255 256 // Minimum time before a frame is due to be rendered before we pull it for 257 // decode. 258 base::TimeDelta min_wait_delta = frame_delay_; 259 base::TimeDelta time_until_render = *render_time - now; 260 if (!next_frame && (time_until_render > min_wait_delta)) { 261 // Example: 262 // We have decoded frame 1 and we have received the complete frame 3, but 263 // not frame 2. If we still have time before frame 3 should be rendered we 264 // will wait for 2 to arrive, however if 2 never show up this timer will hit 265 // and we will pull out frame 3 for decoding and rendering. 266 base::TimeDelta time_until_release = time_until_render - min_wait_delta; 267 cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE, 268 base::Bind(&VideoReceiver::PlayoutTimeout, weak_factory_.GetWeakPtr()), 269 time_until_release); 270 VLOG(1) << "Wait before releasing frame " 271 << static_cast<int>((*encoded_frame)->frame_id) 272 << " time " << time_until_release.InMilliseconds(); 273 return false; 274 } 275 276 base::TimeDelta dont_show_timeout_delta = 277 base::TimeDelta::FromMilliseconds(-kDontShowTimeoutMs); 278 if (codec_ == kVp8 && time_until_render < dont_show_timeout_delta) { 279 (*encoded_frame)->data[0] &= 0xef; 280 VLOG(1) << "Don't show frame " 281 << static_cast<int>((*encoded_frame)->frame_id) 282 << " time_until_render:" << time_until_render.InMilliseconds(); 283 } else { 284 VLOG(1) << "Show frame " 285 << static_cast<int>((*encoded_frame)->frame_id) 286 << " time_until_render:" << time_until_render.InMilliseconds(); 287 } 288 // We have a copy of the frame, release this one. 289 framer_->ReleaseFrame((*encoded_frame)->frame_id); 290 (*encoded_frame)->codec = codec_; 291 return true; 292 } 293 294 void VideoReceiver::PlayoutTimeout() { 295 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 296 if (queued_encoded_callbacks_.empty()) return; 297 298 uint32 rtp_timestamp = 0; 299 bool next_frame = false; 300 scoped_ptr<EncodedVideoFrame> encoded_frame(new EncodedVideoFrame()); 301 302 if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &rtp_timestamp, 303 &next_frame)) { 304 // We have no video frames. Wait for new packet(s). 305 // Since the application can post multiple VideoFrameEncodedCallback and 306 // we only check the next frame to play out we might have multiple timeout 307 // events firing after each other; however this should be a rare event. 308 VLOG(1) << "Failed to retrieved a complete frame at this point in time"; 309 return; 310 } 311 VLOG(1) << "PlayoutTimeout retrieved frame " 312 << static_cast<int>(encoded_frame->frame_id); 313 314 if (decryptor_ && !DecryptVideoFrame(&encoded_frame)) { 315 // Logging already done. 316 return; 317 } 318 319 base::TimeTicks render_time; 320 if (PullEncodedVideoFrame(rtp_timestamp, next_frame, &encoded_frame, 321 &render_time)) { 322 if (!queued_encoded_callbacks_.empty()) { 323 VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front(); 324 queued_encoded_callbacks_.pop_front(); 325 cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, 326 base::Bind(callback, base::Passed(&encoded_frame), render_time)); 327 } 328 } else { 329 // We have a video frame; however we are missing packets and we have time 330 // to wait for new packet(s). 331 } 332 } 333 334 base::TimeTicks VideoReceiver::GetRenderTime(base::TimeTicks now, 335 uint32 rtp_timestamp) { 336 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 337 // Senders time in ms when this frame was captured. 338 // Note: the senders clock and our local clock might not be synced. 339 base::TimeTicks rtp_timestamp_in_ticks; 340 341 if (time_offset_.InMilliseconds() == 0) { 342 if (!rtcp_->RtpTimestampInSenderTime(kVideoFrequency, 343 incoming_rtp_timestamp_, 344 &rtp_timestamp_in_ticks)) { 345 // We have not received any RTCP to sync the stream play it out as soon as 346 // possible. 347 return now; 348 } 349 time_offset_ = time_incoming_packet_ - rtp_timestamp_in_ticks; 350 } else if (time_incoming_packet_updated_) { 351 if (rtcp_->RtpTimestampInSenderTime(kVideoFrequency, 352 incoming_rtp_timestamp_, 353 &rtp_timestamp_in_ticks)) { 354 // Time to update the time_offset. 355 base::TimeDelta time_offset = 356 time_incoming_packet_ - rtp_timestamp_in_ticks; 357 time_offset_ = ((kTimeOffsetFilter - 1) * time_offset_ + time_offset) 358 / kTimeOffsetFilter; 359 } 360 } 361 // Reset |time_incoming_packet_updated_| to enable a future measurement. 362 time_incoming_packet_updated_ = false; 363 if (!rtcp_->RtpTimestampInSenderTime(kVideoFrequency, 364 rtp_timestamp, 365 &rtp_timestamp_in_ticks)) { 366 // This can fail if we have not received any RTCP packets in a long time. 367 return now; 368 } 369 return (rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_); 370 } 371 372 void VideoReceiver::IncomingPacket(const uint8* packet, size_t length, 373 const base::Closure callback) { 374 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 375 if (Rtcp::IsRtcpPacket(packet, length)) { 376 rtcp_->IncomingRtcpPacket(packet, length); 377 } else { 378 rtp_receiver_.ReceivedPacket(packet, length); 379 } 380 cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback); 381 } 382 383 void VideoReceiver::IncomingParsedRtpPacket(const uint8* payload_data, 384 size_t payload_size, 385 const RtpCastHeader& rtp_header) { 386 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 387 388 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); 389 if (time_incoming_packet_.is_null() || now - time_incoming_packet_ > 390 base::TimeDelta::FromMilliseconds(kMinTimeBetweenOffsetUpdatesMs)) { 391 if (time_incoming_packet_.is_null()) InitializeTimers(); 392 incoming_rtp_timestamp_ = rtp_header.webrtc.header.timestamp; 393 time_incoming_packet_ = now; 394 time_incoming_packet_updated_ = true; 395 } 396 397 cast_environment_->Logging()->InsertPacketEvent(kPacketReceived, 398 rtp_header.webrtc.header.timestamp, rtp_header.frame_id, 399 rtp_header.packet_id, rtp_header.max_packet_id, payload_size); 400 401 bool complete = framer_->InsertPacket(payload_data, payload_size, rtp_header); 402 403 if (!complete) return; // Video frame not complete; wait for more packets. 404 if (queued_encoded_callbacks_.empty()) return; // No pending callback. 405 406 VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front(); 407 queued_encoded_callbacks_.pop_front(); 408 cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, 409 base::Bind(&VideoReceiver::GetEncodedVideoFrame, 410 weak_factory_.GetWeakPtr(), callback)); 411 } 412 413 // Send a cast feedback message. Actual message created in the framer (cast 414 // message builder). 415 void VideoReceiver::CastFeedback(const RtcpCastMessage& cast_message) { 416 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 417 // TODO(pwestin): wire up log messages. 418 rtcp_->SendRtcpFromRtpReceiver(&cast_message, NULL); 419 time_last_sent_cast_message_= cast_environment_->Clock()->NowTicks(); 420 } 421 422 // Cast messages should be sent within a maximum interval. Schedule a call 423 // if not triggered elsewhere, e.g. by the cast message_builder. 424 void VideoReceiver::ScheduleNextCastMessage() { 425 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 426 base::TimeTicks send_time; 427 framer_->TimeToSendNextCastMessage(&send_time); 428 429 base::TimeDelta time_to_send = send_time - 430 cast_environment_->Clock()->NowTicks(); 431 time_to_send = std::max(time_to_send, 432 base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); 433 cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE, 434 base::Bind(&VideoReceiver::SendNextCastMessage, 435 weak_factory_.GetWeakPtr()), time_to_send); 436 } 437 438 void VideoReceiver::SendNextCastMessage() { 439 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 440 framer_->SendCastMessage(); // Will only send a message if it is time. 441 ScheduleNextCastMessage(); 442 } 443 444 // Schedule the next RTCP report to be sent back to the sender. 445 void VideoReceiver::ScheduleNextRtcpReport() { 446 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 447 base::TimeDelta time_to_next = rtcp_->TimeToSendNextRtcpReport() - 448 cast_environment_->Clock()->NowTicks(); 449 450 time_to_next = std::max(time_to_next, 451 base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); 452 453 cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE, 454 base::Bind(&VideoReceiver::SendNextRtcpReport, 455 weak_factory_.GetWeakPtr()), time_to_next); 456 } 457 458 void VideoReceiver::SendNextRtcpReport() { 459 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 460 rtcp_->SendRtcpFromRtpReceiver(NULL, NULL); 461 ScheduleNextRtcpReport(); 462 } 463 464 } // namespace cast 465 } // namespace media 466
