Home | History | Annotate | Download | only in video_coding 
      1 /*
      2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/base/checks.h"
     12 #include "webrtc/base/logging.h"
     13 #include "webrtc/base/trace_event.h"
     14 #include "webrtc/common_types.h"
     15 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
     16 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
     17 #include "webrtc/modules/video_coding/encoded_frame.h"
     18 #include "webrtc/modules/video_coding/jitter_buffer.h"
     19 #include "webrtc/modules/video_coding/packet.h"
     20 #include "webrtc/modules/video_coding/video_coding_impl.h"
     21 #include "webrtc/system_wrappers/include/clock.h"
     22 
     23 // #define DEBUG_DECODER_BIT_STREAM
     24 
     25 namespace webrtc {
     26 namespace vcm {
     27 
     28 VideoReceiver::VideoReceiver(Clock* clock, EventFactory* event_factory)
     29     : clock_(clock),
     30       process_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
     31       _receiveCritSect(CriticalSectionWrapper::CreateCriticalSection()),
     32       _timing(clock_),
     33       _receiver(&_timing, clock_, event_factory),
     34       _decodedFrameCallback(&_timing, clock_),
     35       _frameTypeCallback(NULL),
     36       _receiveStatsCallback(NULL),
     37       _decoderTimingCallback(NULL),
     38       _packetRequestCallback(NULL),
     39       render_buffer_callback_(NULL),
     40       _decoder(NULL),
     41 #ifdef DEBUG_DECODER_BIT_STREAM
     42       _bitStreamBeforeDecoder(NULL),
     43 #endif
     44       _frameFromFile(),
     45       _scheduleKeyRequest(false),
     46       max_nack_list_size_(0),
     47       pre_decode_image_callback_(NULL),
     48       _codecDataBase(nullptr, nullptr),
     49       _receiveStatsTimer(1000, clock_),
     50       _retransmissionTimer(10, clock_),
     51       _keyRequestTimer(500, clock_) {
     52   assert(clock_);
     53 #ifdef DEBUG_DECODER_BIT_STREAM
     54   _bitStreamBeforeDecoder = fopen("decoderBitStream.bit", "wb");
     55 #endif
     56 }
     57 
     58 VideoReceiver::~VideoReceiver() {
     59   delete _receiveCritSect;
     60 #ifdef DEBUG_DECODER_BIT_STREAM
     61   fclose(_bitStreamBeforeDecoder);
     62 #endif
     63 }
     64 
     65 int32_t VideoReceiver::Process() {
     66   int32_t returnValue = VCM_OK;
     67 
     68   // Receive-side statistics
     69   if (_receiveStatsTimer.TimeUntilProcess() == 0) {
     70     _receiveStatsTimer.Processed();
     71     CriticalSectionScoped cs(process_crit_sect_.get());
     72     if (_receiveStatsCallback != NULL) {
     73       uint32_t bitRate;
     74       uint32_t frameRate;
     75       _receiver.ReceiveStatistics(&bitRate, &frameRate);
     76       _receiveStatsCallback->OnReceiveRatesUpdated(bitRate, frameRate);
     77     }
     78 
     79     if (_decoderTimingCallback != NULL) {
     80       int decode_ms;
     81       int max_decode_ms;
     82       int current_delay_ms;
     83       int target_delay_ms;
     84       int jitter_buffer_ms;
     85       int min_playout_delay_ms;
     86       int render_delay_ms;
     87       _timing.GetTimings(&decode_ms, &max_decode_ms, &current_delay_ms,
     88                          &target_delay_ms, &jitter_buffer_ms,
     89                          &min_playout_delay_ms, &render_delay_ms);
     90       _decoderTimingCallback->OnDecoderTiming(
     91           decode_ms, max_decode_ms, current_delay_ms, target_delay_ms,
     92           jitter_buffer_ms, min_playout_delay_ms, render_delay_ms);
     93     }
     94 
     95     // Size of render buffer.
     96     if (render_buffer_callback_) {
     97       int buffer_size_ms = _receiver.RenderBufferSizeMs();
     98       render_buffer_callback_->RenderBufferSizeMs(buffer_size_ms);
     99     }
    100   }
    101 
    102   // Key frame requests
    103   if (_keyRequestTimer.TimeUntilProcess() == 0) {
    104     _keyRequestTimer.Processed();
    105     bool request_key_frame = false;
    106     {
    107       CriticalSectionScoped cs(process_crit_sect_.get());
    108       request_key_frame = _scheduleKeyRequest && _frameTypeCallback != NULL;
    109     }
    110     if (request_key_frame) {
    111       const int32_t ret = RequestKeyFrame();
    112       if (ret != VCM_OK && returnValue == VCM_OK) {
    113         returnValue = ret;
    114       }
    115     }
    116   }
    117 
    118   // Packet retransmission requests
    119   // TODO(holmer): Add API for changing Process interval and make sure it's
    120   // disabled when NACK is off.
    121   if (_retransmissionTimer.TimeUntilProcess() == 0) {
    122     _retransmissionTimer.Processed();
    123     bool callback_registered = false;
    124     uint16_t length;
    125     {
    126       CriticalSectionScoped cs(process_crit_sect_.get());
    127       length = max_nack_list_size_;
    128       callback_registered = _packetRequestCallback != NULL;
    129     }
    130     if (callback_registered && length > 0) {
    131       // Collect sequence numbers from the default receiver.
    132       bool request_key_frame = false;
    133       std::vector<uint16_t> nackList = _receiver.NackList(&request_key_frame);
    134       int32_t ret = VCM_OK;
    135       if (request_key_frame) {
    136         ret = RequestKeyFrame();
    137         if (ret != VCM_OK && returnValue == VCM_OK) {
    138           returnValue = ret;
    139         }
    140       }
    141       if (ret == VCM_OK && !nackList.empty()) {
    142         CriticalSectionScoped cs(process_crit_sect_.get());
    143         if (_packetRequestCallback != NULL) {
    144           _packetRequestCallback->ResendPackets(&nackList[0], nackList.size());
    145         }
    146       }
    147     }
    148   }
    149 
    150   return returnValue;
    151 }
    152 
    153 int64_t VideoReceiver::TimeUntilNextProcess() {
    154   int64_t timeUntilNextProcess = _receiveStatsTimer.TimeUntilProcess();
    155   if (_receiver.NackMode() != kNoNack) {
    156     // We need a Process call more often if we are relying on
    157     // retransmissions
    158     timeUntilNextProcess =
    159         VCM_MIN(timeUntilNextProcess, _retransmissionTimer.TimeUntilProcess());
    160   }
    161   timeUntilNextProcess =
    162       VCM_MIN(timeUntilNextProcess, _keyRequestTimer.TimeUntilProcess());
    163 
    164   return timeUntilNextProcess;
    165 }
    166 
    167 int32_t VideoReceiver::SetReceiveChannelParameters(int64_t rtt) {
    168   CriticalSectionScoped receiveCs(_receiveCritSect);
    169   _receiver.UpdateRtt(rtt);
    170   return 0;
    171 }
    172 
    173 // Enable or disable a video protection method.
    174 // Note: This API should be deprecated, as it does not offer a distinction
    175 // between the protection method and decoding with or without errors. If such a
    176 // behavior is desired, use the following API: SetReceiverRobustnessMode.
    177 int32_t VideoReceiver::SetVideoProtection(VCMVideoProtection videoProtection,
    178                                           bool enable) {
    179   // By default, do not decode with errors.
    180   _receiver.SetDecodeErrorMode(kNoErrors);
    181   switch (videoProtection) {
    182     case kProtectionNack: {
    183       RTC_DCHECK(enable);
    184       _receiver.SetNackMode(kNack, -1, -1);
    185       break;
    186     }
    187 
    188     case kProtectionNackFEC: {
    189       CriticalSectionScoped cs(_receiveCritSect);
    190       RTC_DCHECK(enable);
    191       _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1);
    192       _receiver.SetDecodeErrorMode(kNoErrors);
    193       break;
    194     }
    195     case kProtectionFEC:
    196     case kProtectionNone:
    197       // No receiver-side protection.
    198       RTC_DCHECK(enable);
    199       _receiver.SetNackMode(kNoNack, -1, -1);
    200       _receiver.SetDecodeErrorMode(kWithErrors);
    201       break;
    202   }
    203   return VCM_OK;
    204 }
    205 
    206 // Register a receive callback. Will be called whenever there is a new frame
    207 // ready for rendering.
    208 int32_t VideoReceiver::RegisterReceiveCallback(
    209     VCMReceiveCallback* receiveCallback) {
    210   CriticalSectionScoped cs(_receiveCritSect);
    211   _decodedFrameCallback.SetUserReceiveCallback(receiveCallback);
    212   return VCM_OK;
    213 }
    214 
    215 int32_t VideoReceiver::RegisterReceiveStatisticsCallback(
    216     VCMReceiveStatisticsCallback* receiveStats) {
    217   CriticalSectionScoped cs(process_crit_sect_.get());
    218   _receiver.RegisterStatsCallback(receiveStats);
    219   _receiveStatsCallback = receiveStats;
    220   return VCM_OK;
    221 }
    222 
    223 int32_t VideoReceiver::RegisterDecoderTimingCallback(
    224     VCMDecoderTimingCallback* decoderTiming) {
    225   CriticalSectionScoped cs(process_crit_sect_.get());
    226   _decoderTimingCallback = decoderTiming;
    227   return VCM_OK;
    228 }
    229 
    230 // Register an externally defined decoder object.
    231 void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder,
    232                                             uint8_t payloadType) {
    233   CriticalSectionScoped cs(_receiveCritSect);
    234   if (externalDecoder == NULL) {
    235     // Make sure the VCM updates the decoder next time it decodes.
    236     _decoder = NULL;
    237     RTC_CHECK(_codecDataBase.DeregisterExternalDecoder(payloadType));
    238     return;
    239   }
    240   _codecDataBase.RegisterExternalDecoder(externalDecoder, payloadType);
    241 }
    242 
    243 // Register a frame type request callback.
    244 int32_t VideoReceiver::RegisterFrameTypeCallback(
    245     VCMFrameTypeCallback* frameTypeCallback) {
    246   CriticalSectionScoped cs(process_crit_sect_.get());
    247   _frameTypeCallback = frameTypeCallback;
    248   return VCM_OK;
    249 }
    250 
    251 int32_t VideoReceiver::RegisterPacketRequestCallback(
    252     VCMPacketRequestCallback* callback) {
    253   CriticalSectionScoped cs(process_crit_sect_.get());
    254   _packetRequestCallback = callback;
    255   return VCM_OK;
    256 }
    257 
    258 int VideoReceiver::RegisterRenderBufferSizeCallback(
    259     VCMRenderBufferSizeCallback* callback) {
    260   CriticalSectionScoped cs(process_crit_sect_.get());
    261   render_buffer_callback_ = callback;
    262   return VCM_OK;
    263 }
    264 
    265 void VideoReceiver::TriggerDecoderShutdown() {
    266   _receiver.TriggerDecoderShutdown();
    267 }
    268 
    269 // Decode next frame, blocking.
    270 // Should be called as often as possible to get the most out of the decoder.
    271 int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) {
    272   int64_t nextRenderTimeMs;
    273   bool prefer_late_decoding = false;
    274   {
    275     CriticalSectionScoped cs(_receiveCritSect);
    276     prefer_late_decoding = _codecDataBase.PrefersLateDecoding();
    277   }
    278 
    279   VCMEncodedFrame* frame = _receiver.FrameForDecoding(
    280       maxWaitTimeMs, &nextRenderTimeMs, prefer_late_decoding);
    281 
    282   if (!frame)
    283     return VCM_FRAME_NOT_READY;
    284 
    285   CriticalSectionScoped cs(_receiveCritSect);
    286 
    287   // If this frame was too late, we should adjust the delay accordingly
    288   _timing.UpdateCurrentDelay(frame->RenderTimeMs(),
    289                              clock_->TimeInMilliseconds());
    290 
    291   if (pre_decode_image_callback_) {
    292     EncodedImage encoded_image(frame->EncodedImage());
    293     int qp = -1;
    294     if (qp_parser_.GetQp(*frame, &qp)) {
    295       encoded_image.qp_ = qp;
    296     }
    297     pre_decode_image_callback_->Encoded(encoded_image, frame->CodecSpecific(),
    298                                         NULL);
    299   }
    300 
    301 #ifdef DEBUG_DECODER_BIT_STREAM
    302   if (_bitStreamBeforeDecoder != NULL) {
    303     // Write bit stream to file for debugging purposes
    304     if (fwrite(frame->Buffer(), 1, frame->Length(), _bitStreamBeforeDecoder) !=
    305         frame->Length()) {
    306       return -1;
    307     }
    308   }
    309 #endif
    310   const int32_t ret = Decode(*frame);
    311   _receiver.ReleaseFrame(frame);
    312   return ret;
    313 }
    314 
    315 int32_t VideoReceiver::RequestSliceLossIndication(
    316     const uint64_t pictureID) const {
    317   TRACE_EVENT1("webrtc", "RequestSLI", "picture_id", pictureID);
    318   CriticalSectionScoped cs(process_crit_sect_.get());
    319   if (_frameTypeCallback != NULL) {
    320     const int32_t ret =
    321         _frameTypeCallback->SliceLossIndicationRequest(pictureID);
    322     if (ret < 0) {
    323       return ret;
    324     }
    325   } else {
    326     return VCM_MISSING_CALLBACK;
    327   }
    328   return VCM_OK;
    329 }
    330 
    331 int32_t VideoReceiver::RequestKeyFrame() {
    332   TRACE_EVENT0("webrtc", "RequestKeyFrame");
    333   CriticalSectionScoped process_cs(process_crit_sect_.get());
    334   if (_frameTypeCallback != NULL) {
    335     const int32_t ret = _frameTypeCallback->RequestKeyFrame();
    336     if (ret < 0) {
    337       return ret;
    338     }
    339     _scheduleKeyRequest = false;
    340   } else {
    341     return VCM_MISSING_CALLBACK;
    342   }
    343   return VCM_OK;
    344 }
    345 
    346 // Must be called from inside the receive side critical section.
    347 int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) {
    348   TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame.TimeStamp(), "Decode",
    349                           "type", frame.FrameType());
    350   // Change decoder if payload type has changed
    351   _decoder = _codecDataBase.GetDecoder(frame, &_decodedFrameCallback);
    352   if (_decoder == NULL) {
    353     return VCM_NO_CODEC_REGISTERED;
    354   }
    355   // Decode a frame
    356   int32_t ret = _decoder->Decode(frame, clock_->TimeInMilliseconds());
    357 
    358   // Check for failed decoding, run frame type request callback if needed.
    359   bool request_key_frame = false;
    360   if (ret < 0) {
    361     if (ret == VCM_ERROR_REQUEST_SLI) {
    362       return RequestSliceLossIndication(
    363           _decodedFrameCallback.LastReceivedPictureID() + 1);
    364     } else {
    365       request_key_frame = true;
    366     }
    367   } else if (ret == VCM_REQUEST_SLI) {
    368     ret = RequestSliceLossIndication(
    369         _decodedFrameCallback.LastReceivedPictureID() + 1);
    370   }
    371   if (!frame.Complete() || frame.MissingFrame()) {
    372     request_key_frame = true;
    373     ret = VCM_OK;
    374   }
    375   if (request_key_frame) {
    376     CriticalSectionScoped cs(process_crit_sect_.get());
    377     _scheduleKeyRequest = true;
    378   }
    379   TRACE_EVENT_ASYNC_END0("webrtc", "Video", frame.TimeStamp());
    380   return ret;
    381 }
    382 
    383 // Reset the decoder state
    384 int32_t VideoReceiver::ResetDecoder() {
    385   bool reset_key_request = false;
    386   {
    387     CriticalSectionScoped cs(_receiveCritSect);
    388     if (_decoder != NULL) {
    389       _receiver.Reset();
    390       _timing.Reset();
    391       reset_key_request = true;
    392       _decoder->Reset();
    393     }
    394   }
    395   if (reset_key_request) {
    396     CriticalSectionScoped cs(process_crit_sect_.get());
    397     _scheduleKeyRequest = false;
    398   }
    399   return VCM_OK;
    400 }
    401 
    402 // Register possible receive codecs, can be called multiple times
    403 int32_t VideoReceiver::RegisterReceiveCodec(const VideoCodec* receiveCodec,
    404                                             int32_t numberOfCores,
    405                                             bool requireKeyFrame) {
    406   CriticalSectionScoped cs(_receiveCritSect);
    407   if (receiveCodec == NULL) {
    408     return VCM_PARAMETER_ERROR;
    409   }
    410   if (!_codecDataBase.RegisterReceiveCodec(receiveCodec, numberOfCores,
    411                                            requireKeyFrame)) {
    412     return -1;
    413   }
    414   return 0;
    415 }
    416 
    417 // Get current received codec
    418 int32_t VideoReceiver::ReceiveCodec(VideoCodec* currentReceiveCodec) const {
    419   CriticalSectionScoped cs(_receiveCritSect);
    420   if (currentReceiveCodec == NULL) {
    421     return VCM_PARAMETER_ERROR;
    422   }
    423   return _codecDataBase.ReceiveCodec(currentReceiveCodec) ? 0 : -1;
    424 }
    425 
    426 // Get current received codec
    427 VideoCodecType VideoReceiver::ReceiveCodec() const {
    428   CriticalSectionScoped cs(_receiveCritSect);
    429   return _codecDataBase.ReceiveCodec();
    430 }
    431 
    432 // Incoming packet from network parsed and ready for decode, non blocking.
    433 int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload,
    434                                       size_t payloadLength,
    435                                       const WebRtcRTPHeader& rtpInfo) {
    436   if (rtpInfo.frameType == kVideoFrameKey) {
    437     TRACE_EVENT1("webrtc", "VCM::PacketKeyFrame", "seqnum",
    438                  rtpInfo.header.sequenceNumber);
    439   }
    440   if (incomingPayload == NULL) {
    441     // The jitter buffer doesn't handle non-zero payload lengths for packets
    442     // without payload.
    443     // TODO(holmer): We should fix this in the jitter buffer.
    444     payloadLength = 0;
    445   }
    446   const VCMPacket packet(incomingPayload, payloadLength, rtpInfo);
    447   int32_t ret = _receiver.InsertPacket(packet, rtpInfo.type.Video.width,
    448                                        rtpInfo.type.Video.height);
    449   // TODO(holmer): Investigate if this somehow should use the key frame
    450   // request scheduling to throttle the requests.
    451   if (ret == VCM_FLUSH_INDICATOR) {
    452     RequestKeyFrame();
    453     ResetDecoder();
    454   } else if (ret < 0) {
    455     return ret;
    456   }
    457   return VCM_OK;
    458 }
    459 
    460 // Minimum playout delay (used for lip-sync). This is the minimum delay required
    461 // to sync with audio. Not included in  VideoCodingModule::Delay()
    462 // Defaults to 0 ms.
    463 int32_t VideoReceiver::SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) {
    464   _timing.set_min_playout_delay(minPlayoutDelayMs);
    465   return VCM_OK;
    466 }
    467 
    468 // The estimated delay caused by rendering, defaults to
    469 // kDefaultRenderDelayMs = 10 ms
    470 int32_t VideoReceiver::SetRenderDelay(uint32_t timeMS) {
    471   _timing.set_render_delay(timeMS);
    472   return VCM_OK;
    473 }
    474 
    475 // Current video delay
    476 int32_t VideoReceiver::Delay() const {
    477   return _timing.TargetVideoDelay();
    478 }
    479 
    480 uint32_t VideoReceiver::DiscardedPackets() const {
    481   return _receiver.DiscardedPackets();
    482 }
    483 
    484 int VideoReceiver::SetReceiverRobustnessMode(
    485     ReceiverRobustness robustnessMode,
    486     VCMDecodeErrorMode decode_error_mode) {
    487   CriticalSectionScoped cs(_receiveCritSect);
    488   switch (robustnessMode) {
    489     case VideoCodingModule::kNone:
    490       _receiver.SetNackMode(kNoNack, -1, -1);
    491       break;
    492     case VideoCodingModule::kHardNack:
    493       // Always wait for retransmissions (except when decoding with errors).
    494       _receiver.SetNackMode(kNack, -1, -1);
    495       break;
    496     case VideoCodingModule::kSoftNack:
    497 #if 1
    498       assert(false);  // TODO(hlundin): Not completed.
    499       return VCM_NOT_IMPLEMENTED;
    500 #else
    501       // Enable hybrid NACK/FEC. Always wait for retransmissions and don't add
    502       // extra delay when RTT is above kLowRttNackMs.
    503       _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1);
    504       break;
    505 #endif
    506     case VideoCodingModule::kReferenceSelection:
    507 #if 1
    508       assert(false);  // TODO(hlundin): Not completed.
    509       return VCM_NOT_IMPLEMENTED;
    510 #else
    511       if (decode_error_mode == kNoErrors) {
    512         return VCM_PARAMETER_ERROR;
    513       }
    514       _receiver.SetNackMode(kNoNack, -1, -1);
    515       break;
    516 #endif
    517   }
    518   _receiver.SetDecodeErrorMode(decode_error_mode);
    519   return VCM_OK;
    520 }
    521 
    522 void VideoReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) {
    523   CriticalSectionScoped cs(_receiveCritSect);
    524   _receiver.SetDecodeErrorMode(decode_error_mode);
    525 }
    526 
    527 void VideoReceiver::SetNackSettings(size_t max_nack_list_size,
    528                                     int max_packet_age_to_nack,
    529                                     int max_incomplete_time_ms) {
    530   if (max_nack_list_size != 0) {
    531     CriticalSectionScoped process_cs(process_crit_sect_.get());
    532     max_nack_list_size_ = max_nack_list_size;
    533   }
    534   _receiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack,
    535                             max_incomplete_time_ms);
    536 }
    537 
    538 int VideoReceiver::SetMinReceiverDelay(int desired_delay_ms) {
    539   return _receiver.SetMinReceiverDelay(desired_delay_ms);
    540 }
    541 
    542 void VideoReceiver::RegisterPreDecodeImageCallback(
    543     EncodedImageCallback* observer) {
    544   CriticalSectionScoped cs(_receiveCritSect);
    545   pre_decode_image_callback_ = observer;
    546 }
    547 
    548 }  // namespace vcm
    549 }  // namespace webrtc
    550