Home | History | Annotate | Download | only in video_sender 
      1 // Copyright 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "media/cast/video_sender/video_sender.h"
      6 
      7 #include <list>
      8 
      9 #include "base/bind.h"
     10 #include "base/logging.h"
     11 #include "base/message_loop/message_loop.h"
     12 #include "crypto/encryptor.h"
     13 #include "crypto/symmetric_key.h"
     14 #include "media/cast/cast_defines.h"
     15 #include "media/cast/net/pacing/paced_sender.h"
     16 #include "media/cast/video_sender/video_encoder.h"
     17 
     18 namespace media {
     19 namespace cast {
     20 
     21 const int64 kMinSchedulingDelayMs = 1;
     22 
     23 class LocalRtcpVideoSenderFeedback : public RtcpSenderFeedback {
     24  public:
     25   explicit LocalRtcpVideoSenderFeedback(VideoSender* video_sender)
     26       : video_sender_(video_sender) {
     27   }
     28 
     29   virtual void OnReceivedCastFeedback(
     30       const RtcpCastMessage& cast_feedback) OVERRIDE {
     31     video_sender_->OnReceivedCastFeedback(cast_feedback);
     32   }
     33 
     34  private:
     35   VideoSender* video_sender_;
     36 };
     37 
     38 class LocalRtpVideoSenderStatistics : public RtpSenderStatistics {
     39  public:
     40   explicit LocalRtpVideoSenderStatistics(RtpSender* rtp_sender)
     41      : rtp_sender_(rtp_sender) {
     42   }
     43 
     44   virtual void GetStatistics(const base::TimeTicks& now,
     45                              RtcpSenderInfo* sender_info) OVERRIDE {
     46     rtp_sender_->RtpStatistics(now, sender_info);
     47   }
     48 
     49  private:
     50   RtpSender* rtp_sender_;
     51 };
     52 
     53 VideoSender::VideoSender(
     54     scoped_refptr<CastEnvironment> cast_environment,
     55     const VideoSenderConfig& video_config,
     56     VideoEncoderController* const video_encoder_controller,
     57     PacedPacketSender* const paced_packet_sender)
     58     : rtp_max_delay_(
     59           base::TimeDelta::FromMilliseconds(video_config.rtp_max_delay_ms)),
     60       max_frame_rate_(video_config.max_frame_rate),
     61       cast_environment_(cast_environment),
     62       rtcp_feedback_(new LocalRtcpVideoSenderFeedback(this)),
     63       rtp_sender_(new RtpSender(cast_environment, NULL, &video_config,
     64                                 paced_packet_sender)),
     65       last_acked_frame_id_(-1),
     66       last_sent_frame_id_(-1),
     67       duplicate_ack_(0),
     68       last_skip_count_(0),
     69       congestion_control_(cast_environment->Clock(),
     70                           video_config.congestion_control_back_off,
     71                           video_config.max_bitrate,
     72                           video_config.min_bitrate,
     73                           video_config.start_bitrate),
     74       initialized_(false),
     75       weak_factory_(this) {
     76   max_unacked_frames_ = static_cast<uint8>(video_config.rtp_max_delay_ms *
     77       video_config.max_frame_rate / 1000) + 1;
     78   VLOG(1) << "max_unacked_frames " << static_cast<int>(max_unacked_frames_);
     79   DCHECK_GT(max_unacked_frames_, 0) << "Invalid argument";
     80 
     81   rtp_video_sender_statistics_.reset(
     82       new LocalRtpVideoSenderStatistics(rtp_sender_.get()));
     83 
     84   if (video_config.use_external_encoder) {
     85     DCHECK(video_encoder_controller) << "Invalid argument";
     86     video_encoder_controller_ = video_encoder_controller;
     87   } else {
     88     video_encoder_.reset(new VideoEncoder(cast_environment, video_config,
     89         max_unacked_frames_));
     90     video_encoder_controller_ = video_encoder_.get();
     91   }
     92 
     93   if (video_config.aes_iv_mask.size() == kAesKeySize &&
     94       video_config.aes_key.size() == kAesKeySize) {
     95     iv_mask_ = video_config.aes_iv_mask;
     96     crypto::SymmetricKey* key = crypto::SymmetricKey::Import(
     97         crypto::SymmetricKey::AES, video_config.aes_key);
     98     encryptor_.reset(new crypto::Encryptor());
     99     encryptor_->Init(key, crypto::Encryptor::CTR, std::string());
    100   } else if (video_config.aes_iv_mask.size() != 0 ||
    101              video_config.aes_key.size() != 0) {
    102     DCHECK(false) << "Invalid crypto configuration";
    103   }
    104 
    105   rtcp_.reset(new Rtcp(
    106       cast_environment_,
    107       rtcp_feedback_.get(),
    108       paced_packet_sender,
    109       rtp_video_sender_statistics_.get(),
    110       NULL,
    111       video_config.rtcp_mode,
    112       base::TimeDelta::FromMilliseconds(video_config.rtcp_interval),
    113       video_config.sender_ssrc,
    114       video_config.incoming_feedback_ssrc,
    115       video_config.rtcp_c_name));
    116 }
    117 
    118 VideoSender::~VideoSender() {}
    119 
    120 void VideoSender::InitializeTimers() {
    121   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
    122   if (!initialized_) {
    123     initialized_ = true;
    124     ScheduleNextRtcpReport();
    125     ScheduleNextResendCheck();
    126     ScheduleNextSkippedFramesCheck();
    127   }
    128 }
    129 
    130 void VideoSender::InsertRawVideoFrame(
    131     const scoped_refptr<media::VideoFrame>& video_frame,
    132     const base::TimeTicks& capture_time) {
    133   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
    134   DCHECK(video_encoder_.get()) << "Invalid state";
    135   cast_environment_->Logging()->InsertFrameEvent(kVideoFrameReceived,
    136       GetVideoRtpTimestamp(capture_time), kFrameIdUnknown);
    137 
    138   if (!video_encoder_->EncodeVideoFrame(video_frame, capture_time,
    139       base::Bind(&VideoSender::SendEncodedVideoFrameMainThread,
    140           weak_factory_.GetWeakPtr()))) {
    141   }
    142 }
    143 
    144 void VideoSender::InsertCodedVideoFrame(const EncodedVideoFrame* encoded_frame,
    145                                         const base::TimeTicks& capture_time,
    146                                         const base::Closure callback) {
    147   DCHECK(!video_encoder_.get()) << "Invalid state";
    148   DCHECK(encoded_frame) << "Invalid argument";
    149   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
    150 
    151   SendEncodedVideoFrame(encoded_frame, capture_time);
    152   cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback);
    153 }
    154 
    155 void VideoSender::SendEncodedVideoFrameMainThread(
    156     scoped_ptr<EncodedVideoFrame> video_frame,
    157     const base::TimeTicks& capture_time) {
    158   SendEncodedVideoFrame(video_frame.get(), capture_time);
    159 }
    160 
    161 bool VideoSender::EncryptVideoFrame(const EncodedVideoFrame& video_frame,
    162                                     EncodedVideoFrame* encrypted_frame) {
    163   DCHECK(encryptor_) << "Invalid state";
    164 
    165   if (!encryptor_->SetCounter(GetAesNonce(video_frame.frame_id, iv_mask_))) {
    166     NOTREACHED() << "Failed to set counter";
    167     return false;
    168   }
    169 
    170   if (!encryptor_->Encrypt(video_frame.data, &encrypted_frame->data)) {
    171     NOTREACHED() << "Encrypt error";
    172     return false;
    173   }
    174   encrypted_frame->codec = video_frame.codec;
    175   encrypted_frame->key_frame = video_frame.key_frame;
    176   encrypted_frame->frame_id = video_frame.frame_id;
    177   encrypted_frame->last_referenced_frame_id =
    178       video_frame.last_referenced_frame_id;
    179   return true;
    180 }
    181 
    182 void VideoSender::SendEncodedVideoFrame(const EncodedVideoFrame* encoded_frame,
    183                                         const base::TimeTicks& capture_time) {
    184   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
    185   last_send_time_ = cast_environment_->Clock()->NowTicks();
    186 
    187   if (encryptor_) {
    188     EncodedVideoFrame encrypted_video_frame;
    189 
    190     if (!EncryptVideoFrame(*encoded_frame, &encrypted_video_frame)) {
    191       // Logging already done.
    192       return;
    193     }
    194     rtp_sender_->IncomingEncodedVideoFrame(&encrypted_video_frame,
    195                                            capture_time);
    196   } else {
    197     rtp_sender_->IncomingEncodedVideoFrame(encoded_frame, capture_time);
    198   }
    199   if (encoded_frame->key_frame) {
    200     VLOG(1) << "Send encoded key frame; frame_id:"
    201             << static_cast<int>(encoded_frame->frame_id);
    202   }
    203   last_sent_frame_id_ = static_cast<int>(encoded_frame->frame_id);
    204   UpdateFramesInFlight();
    205   InitializeTimers();
    206 }
    207 
    208 void VideoSender::IncomingRtcpPacket(const uint8* packet, size_t length,
    209                                      const base::Closure callback) {
    210   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
    211   rtcp_->IncomingRtcpPacket(packet, length);
    212   cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback);
    213 }
    214 
    215 void VideoSender::ScheduleNextRtcpReport() {
    216   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
    217   base::TimeDelta time_to_next = rtcp_->TimeToSendNextRtcpReport() -
    218       cast_environment_->Clock()->NowTicks();
    219 
    220   time_to_next = std::max(time_to_next,
    221       base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
    222 
    223   cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
    224       base::Bind(&VideoSender::SendRtcpReport, weak_factory_.GetWeakPtr()),
    225                  time_to_next);
    226 }
    227 
    228 void VideoSender::SendRtcpReport() {
    229   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
    230 
    231   RtcpSenderLogMessage sender_log_message;
    232   const FrameRawMap& frame_raw_map =
    233       cast_environment_->Logging()->GetFrameRawData();
    234 
    235   FrameRawMap::const_iterator it = frame_raw_map.begin();
    236   while (it != frame_raw_map.end()) {
    237     RtcpSenderFrameLogMessage frame_message;
    238     frame_message.rtp_timestamp = it->first;
    239     frame_message.frame_status = kRtcpSenderFrameStatusUnknown;
    240     if (it->second.type.empty()) {
    241       ++it;
    242       continue;
    243     }
    244     CastLoggingEvent last_event = it->second.type.back();
    245     switch (last_event) {
    246       case kVideoFrameCaptured:
    247         frame_message.frame_status = kRtcpSenderFrameStatusDroppedByFlowControl;
    248         break;
    249       case kVideoFrameSentToEncoder:
    250         frame_message.frame_status = kRtcpSenderFrameStatusDroppedByEncoder;
    251         break;
    252       case kVideoFrameEncoded:
    253         frame_message.frame_status = kRtcpSenderFrameStatusSentToNetwork;
    254         break;
    255       default:
    256         ++it;
    257         continue;
    258     }
    259     ++it;
    260     if (it == frame_raw_map.end()) {
    261       // Last message on our map; only send if it is kVideoFrameEncoded.
    262       if (last_event != kVideoFrameEncoded) {
    263         // For other events we will wait for it to finish and report the result
    264         // in the next report.
    265         break;
    266       }
    267     }
    268     sender_log_message.push_back(frame_message);
    269   }
    270   rtcp_->SendRtcpFromRtpSender(&sender_log_message);
    271   if (!sender_log_message.empty()) {
    272     VLOG(1) << "Failed to send all log messages";
    273   }
    274 
    275   // TODO(pwestin): When we start pulling out the logging by other means we need
    276   // to synchronize this.
    277   cast_environment_->Logging()->Reset();
    278   ScheduleNextRtcpReport();
    279 }
    280 
    281 void VideoSender::ScheduleNextResendCheck() {
    282   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
    283   base::TimeDelta time_to_next;
    284   if (last_send_time_.is_null()) {
    285     time_to_next = rtp_max_delay_;
    286   } else {
    287     time_to_next = last_send_time_ - cast_environment_->Clock()->NowTicks() +
    288         rtp_max_delay_;
    289   }
    290   time_to_next = std::max(time_to_next,
    291       base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
    292 
    293   cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
    294       base::Bind(&VideoSender::ResendCheck, weak_factory_.GetWeakPtr()),
    295                  time_to_next);
    296 }
    297 
    298 void VideoSender::ResendCheck() {
    299   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
    300   if (!last_send_time_.is_null() && last_sent_frame_id_ != -1) {
    301     base::TimeDelta time_since_last_send =
    302        cast_environment_->Clock()->NowTicks() - last_send_time_;
    303     if (time_since_last_send > rtp_max_delay_) {
    304       if (last_acked_frame_id_ == -1) {
    305         // We have not received any ack, send a key frame.
    306         video_encoder_controller_->GenerateKeyFrame();
    307         last_acked_frame_id_ = -1;
    308         last_sent_frame_id_ = -1;
    309         UpdateFramesInFlight();
    310       } else {
    311         DCHECK_LE(0, last_acked_frame_id_);
    312 
    313         uint32 frame_id = static_cast<uint32>(last_acked_frame_id_ + 1);
    314         VLOG(1) << "ACK timeout resend frame:" << static_cast<int>(frame_id);
    315         ResendFrame(frame_id);
    316       }
    317     }
    318   }
    319   ScheduleNextResendCheck();
    320 }
    321 
    322 void VideoSender::ScheduleNextSkippedFramesCheck() {
    323   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
    324   base::TimeDelta time_to_next;
    325   if (last_checked_skip_count_time_.is_null()) {
    326     time_to_next =
    327         base::TimeDelta::FromMilliseconds(kSkippedFramesCheckPeriodkMs);
    328   } else {
    329     time_to_next = last_checked_skip_count_time_ -
    330         cast_environment_->Clock()->NowTicks() +
    331         base::TimeDelta::FromMilliseconds(kSkippedFramesCheckPeriodkMs);
    332   }
    333   time_to_next = std::max(time_to_next,
    334       base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
    335 
    336   cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
    337       base::Bind(&VideoSender::SkippedFramesCheck, weak_factory_.GetWeakPtr()),
    338                  time_to_next);
    339 }
    340 
    341 void VideoSender::SkippedFramesCheck() {
    342   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
    343   int skip_count = video_encoder_controller_->NumberOfSkippedFrames();
    344   if (skip_count - last_skip_count_ >
    345       kSkippedFramesThreshold * max_frame_rate_) {
    346       // TODO(pwestin): Propagate this up to the application.
    347   }
    348   last_skip_count_ = skip_count;
    349   last_checked_skip_count_time_ = cast_environment_->Clock()->NowTicks();
    350   ScheduleNextSkippedFramesCheck();
    351 }
    352 
    353 void VideoSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) {
    354   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
    355   base::TimeDelta rtt;
    356   base::TimeDelta avg_rtt;
    357   base::TimeDelta min_rtt;
    358   base::TimeDelta max_rtt;
    359 
    360   if (rtcp_->Rtt(&rtt, &avg_rtt, &min_rtt, &max_rtt)) {
    361     cast_environment_->Logging()->InsertGenericEvent(kRttMs,
    362         rtt.InMilliseconds());
    363     // Don't use a RTT lower than our average.
    364     rtt = std::max(rtt, avg_rtt);
    365   } else {
    366     // We have no measured value use default.
    367     rtt = base::TimeDelta::FromMilliseconds(kStartRttMs);
    368   }
    369   if (cast_feedback.missing_frames_and_packets_.empty()) {
    370     // No lost packets.
    371     int resend_frame = -1;
    372     if (last_sent_frame_id_ == -1) return;
    373 
    374     video_encoder_controller_->LatestFrameIdToReference(
    375         cast_feedback.ack_frame_id_);
    376 
    377     if (static_cast<uint32>(last_acked_frame_id_ + 1) ==
    378         cast_feedback.ack_frame_id_) {
    379       uint32 new_bitrate = 0;
    380       if (congestion_control_.OnAck(rtt, &new_bitrate)) {
    381         video_encoder_controller_->SetBitRate(new_bitrate);
    382       }
    383     }
    384     if (static_cast<uint32>(last_acked_frame_id_) == cast_feedback.ack_frame_id_
    385         // We only count duplicate ACKs when we have sent newer frames.
    386         && IsNewerFrameId(last_sent_frame_id_, last_acked_frame_id_)) {
    387       duplicate_ack_++;
    388     } else {
    389       duplicate_ack_ = 0;
    390     }
    391     if (duplicate_ack_ >= 2 && duplicate_ack_ % 3 == 2) {
    392       // Resend last ACK + 1 frame.
    393       resend_frame = static_cast<uint32>(last_acked_frame_id_ + 1);
    394     }
    395     if (resend_frame != -1) {
    396       DCHECK_LE(0, resend_frame);
    397       VLOG(1) << "Received duplicate ACK for frame:"
    398               << static_cast<int>(resend_frame);
    399       ResendFrame(static_cast<uint32>(resend_frame));
    400     }
    401   } else {
    402     rtp_sender_->ResendPackets(cast_feedback.missing_frames_and_packets_);
    403     last_send_time_ = cast_environment_->Clock()->NowTicks();
    404 
    405     uint32 new_bitrate = 0;
    406     if (congestion_control_.OnNack(rtt, &new_bitrate)) {
    407       video_encoder_controller_->SetBitRate(new_bitrate);
    408     }
    409   }
    410   ReceivedAck(cast_feedback.ack_frame_id_);
    411 }
    412 
    413 void VideoSender::ReceivedAck(uint32 acked_frame_id) {
    414   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
    415   last_acked_frame_id_ = static_cast<int>(acked_frame_id);
    416   cast_environment_->Logging()->InsertGenericEvent(kAckReceived,
    417                                                    acked_frame_id);
    418   VLOG(1) << "ReceivedAck:" << static_cast<int>(acked_frame_id);
    419   last_acked_frame_id_ = acked_frame_id;
    420   UpdateFramesInFlight();
    421 }
    422 
    423 void VideoSender::UpdateFramesInFlight() {
    424   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
    425   if (last_sent_frame_id_ != -1) {
    426     DCHECK_LE(0, last_sent_frame_id_);
    427     uint32 frames_in_flight;
    428     if (last_acked_frame_id_ != -1) {
    429       DCHECK_LE(0, last_acked_frame_id_);
    430       frames_in_flight = static_cast<uint32>(last_sent_frame_id_) -
    431                          static_cast<uint32>(last_acked_frame_id_);
    432     } else {
    433       frames_in_flight = static_cast<uint32>(last_sent_frame_id_) + 1;
    434     }
    435     VLOG(1) << "Frames in flight; last sent: " << last_sent_frame_id_
    436             << " last acked:" << last_acked_frame_id_;
    437     if (frames_in_flight >= max_unacked_frames_) {
    438       video_encoder_controller_->SkipNextFrame(true);
    439       return;
    440     }
    441   }
    442   video_encoder_controller_->SkipNextFrame(false);
    443 }
    444 
    445 void VideoSender::ResendFrame(uint32 resend_frame_id) {
    446   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
    447   MissingFramesAndPacketsMap missing_frames_and_packets;
    448   PacketIdSet missing;
    449   missing_frames_and_packets.insert(std::make_pair(resend_frame_id, missing));
    450   rtp_sender_->ResendPackets(missing_frames_and_packets);
    451   last_send_time_ = cast_environment_->Clock()->NowTicks();
    452 }
    453 
    454 }  // namespace cast
    455 }  // namespace media
    456