Home | History | Annotate | Download | only in source 
      1 /*
      2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/modules/video_coding/main/source/receiver.h"
     12 
     13 #include <assert.h>
     14 
     15 #include <cstdlib>
     16 
     17 #include "webrtc/modules/video_coding/main/source/encoded_frame.h"
     18 #include "webrtc/modules/video_coding/main/source/internal_defines.h"
     19 #include "webrtc/modules/video_coding/main/source/media_opt_util.h"
     20 #include "webrtc/system_wrappers/interface/clock.h"
     21 #include "webrtc/system_wrappers/interface/logging.h"
     22 #include "webrtc/system_wrappers/interface/trace_event.h"
     23 
     24 namespace webrtc {
     25 
     26 enum { kMaxReceiverDelayMs = 10000 };
     27 
     28 VCMReceiver::VCMReceiver(VCMTiming* timing,
     29                          Clock* clock,
     30                          EventFactory* event_factory,
     31                          bool master)
     32     : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
     33       clock_(clock),
     34       master_(master),
     35       jitter_buffer_(clock_, event_factory),
     36       timing_(timing),
     37       render_wait_event_(event_factory->CreateEvent()),
     38       state_(kPassive),
     39       max_video_delay_ms_(kMaxVideoDelayMs) {}
     40 
     41 VCMReceiver::~VCMReceiver() {
     42   render_wait_event_->Set();
     43   delete crit_sect_;
     44 }
     45 
     46 void VCMReceiver::Reset() {
     47   CriticalSectionScoped cs(crit_sect_);
     48   if (!jitter_buffer_.Running()) {
     49     jitter_buffer_.Start();
     50   } else {
     51     jitter_buffer_.Flush();
     52   }
     53   render_wait_event_->Reset();
     54   if (master_) {
     55     state_ = kReceiving;
     56   } else {
     57     state_ = kPassive;
     58   }
     59 }
     60 
     61 int32_t VCMReceiver::Initialize() {
     62   Reset();
     63   CriticalSectionScoped cs(crit_sect_);
     64   if (!master_) {
     65     SetNackMode(kNoNack, -1, -1);
     66   }
     67   return VCM_OK;
     68 }
     69 
     70 void VCMReceiver::UpdateRtt(uint32_t rtt) {
     71   jitter_buffer_.UpdateRtt(rtt);
     72 }
     73 
     74 int32_t VCMReceiver::InsertPacket(const VCMPacket& packet,
     75                                   uint16_t frame_width,
     76                                   uint16_t frame_height) {
     77   // Insert the packet into the jitter buffer. The packet can either be empty or
     78   // contain media at this point.
     79   bool retransmitted = false;
     80   const VCMFrameBufferEnum ret = jitter_buffer_.InsertPacket(packet,
     81                                                              &retransmitted);
     82   if (ret == kOldPacket) {
     83     return VCM_OK;
     84   } else if (ret == kFlushIndicator) {
     85     return VCM_FLUSH_INDICATOR;
     86   } else if (ret < 0) {
     87     return VCM_JITTER_BUFFER_ERROR;
     88   }
     89   if (ret == kCompleteSession && !retransmitted) {
     90     // We don't want to include timestamps which have suffered from
     91     // retransmission here, since we compensate with extra retransmission
     92     // delay within the jitter estimate.
     93     timing_->IncomingTimestamp(packet.timestamp, clock_->TimeInMilliseconds());
     94   }
     95   return VCM_OK;
     96 }
     97 
     98 VCMEncodedFrame* VCMReceiver::FrameForDecoding(
     99     uint16_t max_wait_time_ms,
    100     int64_t& next_render_time_ms,
    101     bool render_timing,
    102     VCMReceiver* dual_receiver) {
    103   const int64_t start_time_ms = clock_->TimeInMilliseconds();
    104   uint32_t frame_timestamp = 0;
    105   // Exhaust wait time to get a complete frame for decoding.
    106   bool found_frame = jitter_buffer_.NextCompleteTimestamp(
    107       max_wait_time_ms, &frame_timestamp);
    108 
    109   if (!found_frame) {
    110     // Get an incomplete frame when enabled.
    111     const bool dual_receiver_enabled_and_passive = (dual_receiver != NULL &&
    112         dual_receiver->State() == kPassive &&
    113         dual_receiver->NackMode() == kNack);
    114     if (dual_receiver_enabled_and_passive &&
    115         !jitter_buffer_.CompleteSequenceWithNextFrame()) {
    116       // Jitter buffer state might get corrupt with this frame.
    117       dual_receiver->CopyJitterBufferStateFromReceiver(*this);
    118     }
    119     found_frame = jitter_buffer_.NextMaybeIncompleteTimestamp(
    120         &frame_timestamp);
    121   }
    122 
    123   if (!found_frame) {
    124     return NULL;
    125   }
    126 
    127   // We have a frame - Set timing and render timestamp.
    128   timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs());
    129   const int64_t now_ms = clock_->TimeInMilliseconds();
    130   timing_->UpdateCurrentDelay(frame_timestamp);
    131   next_render_time_ms = timing_->RenderTimeMs(frame_timestamp, now_ms);
    132   // Check render timing.
    133   bool timing_error = false;
    134   // Assume that render timing errors are due to changes in the video stream.
    135   if (next_render_time_ms < 0) {
    136     timing_error = true;
    137   } else if (std::abs(next_render_time_ms - now_ms) > max_video_delay_ms_) {
    138     int frame_delay = static_cast<int>(std::abs(next_render_time_ms - now_ms));
    139     LOG(LS_WARNING) << "A frame about to be decoded is out of the configured "
    140                     << "delay bounds (" << frame_delay << " > "
    141                     << max_video_delay_ms_
    142                     << "). Resetting the video jitter buffer.";
    143     timing_error = true;
    144   } else if (static_cast<int>(timing_->TargetVideoDelay()) >
    145              max_video_delay_ms_) {
    146     LOG(LS_WARNING) << "The video target delay has grown larger than "
    147                     << max_video_delay_ms_ << " ms. Resetting jitter buffer.";
    148     timing_error = true;
    149   }
    150 
    151   if (timing_error) {
    152     // Timing error => reset timing and flush the jitter buffer.
    153     jitter_buffer_.Flush();
    154     timing_->Reset();
    155     return NULL;
    156   }
    157 
    158   if (!render_timing) {
    159     // Decode frame as close as possible to the render timestamp.
    160     const int32_t available_wait_time = max_wait_time_ms -
    161         static_cast<int32_t>(clock_->TimeInMilliseconds() - start_time_ms);
    162     uint16_t new_max_wait_time = static_cast<uint16_t>(
    163         VCM_MAX(available_wait_time, 0));
    164     uint32_t wait_time_ms = timing_->MaxWaitingTime(
    165         next_render_time_ms, clock_->TimeInMilliseconds());
    166     if (new_max_wait_time < wait_time_ms) {
    167       // We're not allowed to wait until the frame is supposed to be rendered,
    168       // waiting as long as we're allowed to avoid busy looping, and then return
    169       // NULL. Next call to this function might return the frame.
    170       render_wait_event_->Wait(max_wait_time_ms);
    171       return NULL;
    172     }
    173     // Wait until it's time to render.
    174     render_wait_event_->Wait(wait_time_ms);
    175   }
    176 
    177   // Extract the frame from the jitter buffer and set the render time.
    178   VCMEncodedFrame* frame = jitter_buffer_.ExtractAndSetDecode(frame_timestamp);
    179   if (frame == NULL) {
    180     return NULL;
    181   }
    182   frame->SetRenderTime(next_render_time_ms);
    183   TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame->TimeStamp(),
    184                           "SetRenderTS", "render_time", next_render_time_ms);
    185   if (dual_receiver != NULL) {
    186     dual_receiver->UpdateState(*frame);
    187   }
    188   if (!frame->Complete()) {
    189     // Update stats for incomplete frames.
    190     bool retransmitted = false;
    191     const int64_t last_packet_time_ms =
    192         jitter_buffer_.LastPacketTime(frame, &retransmitted);
    193     if (last_packet_time_ms >= 0 && !retransmitted) {
    194       // We don't want to include timestamps which have suffered from
    195       // retransmission here, since we compensate with extra retransmission
    196       // delay within the jitter estimate.
    197       timing_->IncomingTimestamp(frame_timestamp, last_packet_time_ms);
    198     }
    199   }
    200   return frame;
    201 }
    202 
    203 void VCMReceiver::ReleaseFrame(VCMEncodedFrame* frame) {
    204   jitter_buffer_.ReleaseFrame(frame);
    205 }
    206 
    207 void VCMReceiver::ReceiveStatistics(uint32_t* bitrate,
    208                                     uint32_t* framerate) {
    209   assert(bitrate);
    210   assert(framerate);
    211   jitter_buffer_.IncomingRateStatistics(framerate, bitrate);
    212 }
    213 
    214 void VCMReceiver::ReceivedFrameCount(VCMFrameCount* frame_count) const {
    215   assert(frame_count);
    216   std::map<FrameType, uint32_t> counts(jitter_buffer_.FrameStatistics());
    217   frame_count->numDeltaFrames = counts[kVideoFrameDelta];
    218   frame_count->numKeyFrames = counts[kVideoFrameKey];
    219 }
    220 
    221 uint32_t VCMReceiver::DiscardedPackets() const {
    222   return jitter_buffer_.num_discarded_packets();
    223 }
    224 
    225 void VCMReceiver::SetNackMode(VCMNackMode nackMode,
    226                               int low_rtt_nack_threshold_ms,
    227                               int high_rtt_nack_threshold_ms) {
    228   CriticalSectionScoped cs(crit_sect_);
    229   // Default to always having NACK enabled in hybrid mode.
    230   jitter_buffer_.SetNackMode(nackMode, low_rtt_nack_threshold_ms,
    231                              high_rtt_nack_threshold_ms);
    232   if (!master_) {
    233     state_ = kPassive;  // The dual decoder defaults to passive.
    234   }
    235 }
    236 
    237 void VCMReceiver::SetNackSettings(size_t max_nack_list_size,
    238                                   int max_packet_age_to_nack,
    239                                   int max_incomplete_time_ms) {
    240   jitter_buffer_.SetNackSettings(max_nack_list_size,
    241                                  max_packet_age_to_nack,
    242                                  max_incomplete_time_ms);
    243 }
    244 
    245 VCMNackMode VCMReceiver::NackMode() const {
    246   CriticalSectionScoped cs(crit_sect_);
    247   return jitter_buffer_.nack_mode();
    248 }
    249 
    250 VCMNackStatus VCMReceiver::NackList(uint16_t* nack_list,
    251                                     uint16_t size,
    252                                     uint16_t* nack_list_length) {
    253   bool request_key_frame = false;
    254   uint16_t* internal_nack_list = jitter_buffer_.GetNackList(
    255       nack_list_length, &request_key_frame);
    256   assert(*nack_list_length <= size);
    257   if (internal_nack_list != NULL && *nack_list_length > 0) {
    258     memcpy(nack_list, internal_nack_list, *nack_list_length * sizeof(uint16_t));
    259   }
    260   if (request_key_frame) {
    261     return kNackKeyFrameRequest;
    262   }
    263   return kNackOk;
    264 }
    265 
    266 // Decide whether we should change decoder state. This should be done if the
    267 // dual decoder has caught up with the decoder decoding with packet losses.
    268 bool VCMReceiver::DualDecoderCaughtUp(VCMEncodedFrame* dual_frame,
    269                                       VCMReceiver& dual_receiver) const {
    270   if (dual_frame == NULL) {
    271     return false;
    272   }
    273   if (jitter_buffer_.LastDecodedTimestamp() == dual_frame->TimeStamp()) {
    274     dual_receiver.UpdateState(kWaitForPrimaryDecode);
    275     return true;
    276   }
    277   return false;
    278 }
    279 
    280 void VCMReceiver::CopyJitterBufferStateFromReceiver(
    281     const VCMReceiver& receiver) {
    282   jitter_buffer_.CopyFrom(receiver.jitter_buffer_);
    283 }
    284 
    285 VCMReceiverState VCMReceiver::State() const {
    286   CriticalSectionScoped cs(crit_sect_);
    287   return state_;
    288 }
    289 
    290 void VCMReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) {
    291   jitter_buffer_.SetDecodeErrorMode(decode_error_mode);
    292 }
    293 
    294 VCMDecodeErrorMode VCMReceiver::DecodeErrorMode() const {
    295   return jitter_buffer_.decode_error_mode();
    296 }
    297 
    298 int VCMReceiver::SetMinReceiverDelay(int desired_delay_ms) {
    299   CriticalSectionScoped cs(crit_sect_);
    300   if (desired_delay_ms < 0 || desired_delay_ms > kMaxReceiverDelayMs) {
    301     return -1;
    302   }
    303   max_video_delay_ms_ = desired_delay_ms + kMaxVideoDelayMs;
    304   // Initializing timing to the desired delay.
    305   timing_->set_min_playout_delay(desired_delay_ms);
    306   return 0;
    307 }
    308 
    309 int VCMReceiver::RenderBufferSizeMs() {
    310   uint32_t timestamp_start = 0u;
    311   uint32_t timestamp_end = 0u;
    312   // Render timestamps are computed just prior to decoding. Therefore this is
    313   // only an estimate based on frames' timestamps and current timing state.
    314   jitter_buffer_.RenderBufferSize(&timestamp_start, &timestamp_end);
    315   if (timestamp_start == timestamp_end) {
    316     return 0;
    317   }
    318   // Update timing.
    319   const int64_t now_ms = clock_->TimeInMilliseconds();
    320   timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs());
    321   // Get render timestamps.
    322   uint32_t render_start = timing_->RenderTimeMs(timestamp_start, now_ms);
    323   uint32_t render_end = timing_->RenderTimeMs(timestamp_end, now_ms);
    324   return render_end - render_start;
    325 }
    326 
    327 void VCMReceiver::UpdateState(VCMReceiverState new_state) {
    328   CriticalSectionScoped cs(crit_sect_);
    329   assert(!(state_ == kPassive && new_state == kWaitForPrimaryDecode));
    330   state_ = new_state;
    331 }
    332 
    333 void VCMReceiver::UpdateState(const VCMEncodedFrame& frame) {
    334   if (jitter_buffer_.nack_mode() == kNoNack) {
    335     // Dual decoder mode has not been enabled.
    336     return;
    337   }
    338   // Update the dual receiver state.
    339   if (frame.Complete() && frame.FrameType() == kVideoFrameKey) {
    340     UpdateState(kPassive);
    341   }
    342   if (State() == kWaitForPrimaryDecode &&
    343       frame.Complete() && !frame.MissingFrame()) {
    344     UpdateState(kPassive);
    345   }
    346   if (frame.MissingFrame() || !frame.Complete()) {
    347     // State was corrupted, enable dual receiver.
    348     UpdateState(kReceiving);
    349   }
    350 }
    351 }  // namespace webrtc
    352