Home | History | Annotate | Download | only in video_coding 
      1 /*
      2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 
     12 #include <algorithm>  // std::max
     13 
     14 #include "webrtc/base/checks.h"
     15 #include "webrtc/base/logging.h"
     16 #include "webrtc/common_types.h"
     17 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
     18 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
     19 #include "webrtc/modules/video_coding/encoded_frame.h"
     20 #include "webrtc/modules/video_coding/utility/quality_scaler.h"
     21 #include "webrtc/modules/video_coding/video_coding_impl.h"
     22 #include "webrtc/system_wrappers/include/clock.h"
     23 
     24 namespace webrtc {
     25 namespace vcm {
     26 
     27 VideoSender::VideoSender(Clock* clock,
     28                          EncodedImageCallback* post_encode_callback,
     29                          VideoEncoderRateObserver* encoder_rate_observer,
     30                          VCMQMSettingsCallback* qm_settings_callback)
     31     : clock_(clock),
     32       process_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
     33       _encoder(nullptr),
     34       _encodedFrameCallback(post_encode_callback),
     35       _nextFrameTypes(1, kVideoFrameDelta),
     36       _mediaOpt(clock_),
     37       _sendStatsCallback(nullptr),
     38       _codecDataBase(encoder_rate_observer, &_encodedFrameCallback),
     39       frame_dropper_enabled_(true),
     40       _sendStatsTimer(1000, clock_),
     41       current_codec_(),
     42       qm_settings_callback_(qm_settings_callback),
     43       protection_callback_(nullptr),
     44       encoder_params_({0, 0, 0, 0}) {
     45   // Allow VideoSender to be created on one thread but used on another, post
     46   // construction. This is currently how this class is being used by at least
     47   // one external project (diffractor).
     48   _mediaOpt.EnableQM(qm_settings_callback_ != nullptr);
     49   _mediaOpt.Reset();
     50   main_thread_.DetachFromThread();
     51 }
     52 
     53 VideoSender::~VideoSender() {}
     54 
     55 int32_t VideoSender::Process() {
     56   int32_t returnValue = VCM_OK;
     57 
     58   if (_sendStatsTimer.TimeUntilProcess() == 0) {
     59     _sendStatsTimer.Processed();
     60     CriticalSectionScoped cs(process_crit_sect_.get());
     61     if (_sendStatsCallback != nullptr) {
     62       uint32_t bitRate = _mediaOpt.SentBitRate();
     63       uint32_t frameRate = _mediaOpt.SentFrameRate();
     64       _sendStatsCallback->SendStatistics(bitRate, frameRate);
     65     }
     66   }
     67 
     68   {
     69     rtc::CritScope cs(&params_lock_);
     70     // Force an encoder parameters update, so that incoming frame rate is
     71     // updated even if bandwidth hasn't changed.
     72     encoder_params_.input_frame_rate = _mediaOpt.InputFrameRate();
     73   }
     74 
     75   return returnValue;
     76 }
     77 
     78 int64_t VideoSender::TimeUntilNextProcess() {
     79   return _sendStatsTimer.TimeUntilProcess();
     80 }
     81 
     82 // Register the send codec to be used.
     83 int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec,
     84                                        uint32_t numberOfCores,
     85                                        uint32_t maxPayloadSize) {
     86   RTC_DCHECK(main_thread_.CalledOnValidThread());
     87   rtc::CritScope lock(&send_crit_);
     88   if (sendCodec == nullptr) {
     89     return VCM_PARAMETER_ERROR;
     90   }
     91 
     92   bool ret =
     93       _codecDataBase.SetSendCodec(sendCodec, numberOfCores, maxPayloadSize);
     94 
     95   // Update encoder regardless of result to make sure that we're not holding on
     96   // to a deleted instance.
     97   _encoder = _codecDataBase.GetEncoder();
     98   // Cache the current codec here so they can be fetched from this thread
     99   // without requiring the _sendCritSect lock.
    100   current_codec_ = *sendCodec;
    101 
    102   if (!ret) {
    103     LOG(LS_ERROR) << "Failed to initialize set encoder with payload name '"
    104                   << sendCodec->plName << "'.";
    105     return VCM_CODEC_ERROR;
    106   }
    107 
    108   int numLayers;
    109   if (sendCodec->codecType == kVideoCodecVP8) {
    110     numLayers = sendCodec->codecSpecific.VP8.numberOfTemporalLayers;
    111   } else if (sendCodec->codecType == kVideoCodecVP9) {
    112     numLayers = sendCodec->codecSpecific.VP9.numberOfTemporalLayers;
    113   } else {
    114     numLayers = 1;
    115   }
    116 
    117   // If we have screensharing and we have layers, we disable frame dropper.
    118   bool disable_frame_dropper =
    119       numLayers > 1 && sendCodec->mode == kScreensharing;
    120   if (disable_frame_dropper) {
    121     _mediaOpt.EnableFrameDropper(false);
    122   } else if (frame_dropper_enabled_) {
    123     _mediaOpt.EnableFrameDropper(true);
    124   }
    125   _nextFrameTypes.clear();
    126   _nextFrameTypes.resize(VCM_MAX(sendCodec->numberOfSimulcastStreams, 1),
    127                          kVideoFrameDelta);
    128 
    129   _mediaOpt.SetEncodingData(sendCodec->codecType, sendCodec->maxBitrate * 1000,
    130                             sendCodec->startBitrate * 1000, sendCodec->width,
    131                             sendCodec->height, sendCodec->maxFramerate,
    132                             numLayers, maxPayloadSize);
    133   return VCM_OK;
    134 }
    135 
    136 // Register an external decoder object.
    137 // This can not be used together with external decoder callbacks.
    138 void VideoSender::RegisterExternalEncoder(VideoEncoder* externalEncoder,
    139                                           uint8_t payloadType,
    140                                           bool internalSource /*= false*/) {
    141   RTC_DCHECK(main_thread_.CalledOnValidThread());
    142 
    143   rtc::CritScope lock(&send_crit_);
    144 
    145   if (externalEncoder == nullptr) {
    146     bool wasSendCodec = false;
    147     RTC_CHECK(
    148         _codecDataBase.DeregisterExternalEncoder(payloadType, &wasSendCodec));
    149     if (wasSendCodec) {
    150       // Make sure the VCM doesn't use the de-registered codec
    151       _encoder = nullptr;
    152     }
    153     return;
    154   }
    155   _codecDataBase.RegisterExternalEncoder(externalEncoder, payloadType,
    156                                          internalSource);
    157 }
    158 
    159 // Get encode bitrate
    160 int VideoSender::Bitrate(unsigned int* bitrate) const {
    161   RTC_DCHECK(main_thread_.CalledOnValidThread());
    162   // Since we're running on the thread that's the only thread known to modify
    163   // the value of _encoder, we don't need to grab the lock here.
    164 
    165   if (!_encoder)
    166     return VCM_UNINITIALIZED;
    167   *bitrate = _encoder->GetEncoderParameters().target_bitrate;
    168   return 0;
    169 }
    170 
    171 // Get encode frame rate
    172 int VideoSender::FrameRate(unsigned int* framerate) const {
    173   RTC_DCHECK(main_thread_.CalledOnValidThread());
    174   // Since we're running on the thread that's the only thread known to modify
    175   // the value of _encoder, we don't need to grab the lock here.
    176 
    177   if (!_encoder)
    178     return VCM_UNINITIALIZED;
    179 
    180   *framerate = _encoder->GetEncoderParameters().input_frame_rate;
    181   return 0;
    182 }
    183 
    184 int32_t VideoSender::SetChannelParameters(uint32_t target_bitrate,
    185                                           uint8_t lossRate,
    186                                           int64_t rtt) {
    187   uint32_t target_rate =
    188       _mediaOpt.SetTargetRates(target_bitrate, lossRate, rtt,
    189                                protection_callback_, qm_settings_callback_);
    190 
    191   uint32_t input_frame_rate = _mediaOpt.InputFrameRate();
    192 
    193   rtc::CritScope cs(&params_lock_);
    194   encoder_params_ = {target_rate, lossRate, rtt, input_frame_rate};
    195 
    196   return VCM_OK;
    197 }
    198 
    199 void VideoSender::SetEncoderParameters(EncoderParameters params) {
    200   if (params.target_bitrate == 0)
    201     return;
    202 
    203   if (params.input_frame_rate == 0) {
    204     // No frame rate estimate available, use default.
    205     params.input_frame_rate = current_codec_.maxFramerate;
    206   }
    207   if (_encoder != nullptr)
    208     _encoder->SetEncoderParameters(params);
    209 }
    210 
    211 int32_t VideoSender::RegisterTransportCallback(
    212     VCMPacketizationCallback* transport) {
    213   rtc::CritScope lock(&send_crit_);
    214   _encodedFrameCallback.SetMediaOpt(&_mediaOpt);
    215   _encodedFrameCallback.SetTransportCallback(transport);
    216   return VCM_OK;
    217 }
    218 
    219 // Register video output information callback which will be called to deliver
    220 // information about the video stream produced by the encoder, for instance the
    221 // average frame rate and bit rate.
    222 int32_t VideoSender::RegisterSendStatisticsCallback(
    223     VCMSendStatisticsCallback* sendStats) {
    224   CriticalSectionScoped cs(process_crit_sect_.get());
    225   _sendStatsCallback = sendStats;
    226   return VCM_OK;
    227 }
    228 
    229 // Register a video protection callback which will be called to deliver the
    230 // requested FEC rate and NACK status (on/off).
    231 // Note: this callback is assumed to only be registered once and before it is
    232 // used in this class.
    233 int32_t VideoSender::RegisterProtectionCallback(
    234     VCMProtectionCallback* protection_callback) {
    235   RTC_DCHECK(protection_callback == nullptr || protection_callback_ == nullptr);
    236   protection_callback_ = protection_callback;
    237   return VCM_OK;
    238 }
    239 
    240 // Enable or disable a video protection method.
    241 void VideoSender::SetVideoProtection(VCMVideoProtection videoProtection) {
    242   rtc::CritScope lock(&send_crit_);
    243   switch (videoProtection) {
    244     case kProtectionNone:
    245       _mediaOpt.SetProtectionMethod(media_optimization::kNone);
    246       break;
    247     case kProtectionNack:
    248       _mediaOpt.SetProtectionMethod(media_optimization::kNack);
    249       break;
    250     case kProtectionNackFEC:
    251       _mediaOpt.SetProtectionMethod(media_optimization::kNackFec);
    252       break;
    253     case kProtectionFEC:
    254       _mediaOpt.SetProtectionMethod(media_optimization::kFec);
    255       break;
    256   }
    257 }
    258 // Add one raw video frame to the encoder, blocking.
    259 int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame,
    260                                    const VideoContentMetrics* contentMetrics,
    261                                    const CodecSpecificInfo* codecSpecificInfo) {
    262   EncoderParameters encoder_params;
    263   {
    264     rtc::CritScope lock(&params_lock_);
    265     encoder_params = encoder_params_;
    266   }
    267   rtc::CritScope lock(&send_crit_);
    268   if (_encoder == nullptr)
    269     return VCM_UNINITIALIZED;
    270   SetEncoderParameters(encoder_params);
    271   // TODO(holmer): Add support for dropping frames per stream. Currently we
    272   // only have one frame dropper for all streams.
    273   if (_nextFrameTypes[0] == kEmptyFrame) {
    274     return VCM_OK;
    275   }
    276   if (_mediaOpt.DropFrame()) {
    277     _encoder->OnDroppedFrame();
    278     return VCM_OK;
    279   }
    280   _mediaOpt.UpdateContentData(contentMetrics);
    281   // TODO(pbos): Make sure setting send codec is synchronized with video
    282   // processing so frame size always matches.
    283   if (!_codecDataBase.MatchesCurrentResolution(videoFrame.width(),
    284                                                videoFrame.height())) {
    285     LOG(LS_ERROR) << "Incoming frame doesn't match set resolution. Dropping.";
    286     return VCM_PARAMETER_ERROR;
    287   }
    288   VideoFrame converted_frame = videoFrame;
    289   if (converted_frame.native_handle() && !_encoder->SupportsNativeHandle()) {
    290     // This module only supports software encoding.
    291     // TODO(pbos): Offload conversion from the encoder thread.
    292     converted_frame = converted_frame.ConvertNativeToI420Frame();
    293     RTC_CHECK(!converted_frame.IsZeroSize())
    294         << "Frame conversion failed, won't be able to encode frame.";
    295   }
    296   int32_t ret =
    297       _encoder->Encode(converted_frame, codecSpecificInfo, _nextFrameTypes);
    298   if (ret < 0) {
    299     LOG(LS_ERROR) << "Failed to encode frame. Error code: " << ret;
    300     return ret;
    301   }
    302   for (size_t i = 0; i < _nextFrameTypes.size(); ++i) {
    303     _nextFrameTypes[i] = kVideoFrameDelta;  // Default frame type.
    304   }
    305   if (qm_settings_callback_)
    306     qm_settings_callback_->SetTargetFramerate(_encoder->GetTargetFramerate());
    307   return VCM_OK;
    308 }
    309 
    310 int32_t VideoSender::IntraFrameRequest(int stream_index) {
    311   rtc::CritScope lock(&send_crit_);
    312   if (stream_index < 0 ||
    313       static_cast<unsigned int>(stream_index) >= _nextFrameTypes.size()) {
    314     return -1;
    315   }
    316   _nextFrameTypes[stream_index] = kVideoFrameKey;
    317   if (_encoder != nullptr && _encoder->InternalSource()) {
    318     // Try to request the frame if we have an external encoder with
    319     // internal source since AddVideoFrame never will be called.
    320     if (_encoder->RequestFrame(_nextFrameTypes) == WEBRTC_VIDEO_CODEC_OK) {
    321       _nextFrameTypes[stream_index] = kVideoFrameDelta;
    322     }
    323   }
    324   return VCM_OK;
    325 }
    326 
    327 int32_t VideoSender::EnableFrameDropper(bool enable) {
    328   rtc::CritScope lock(&send_crit_);
    329   frame_dropper_enabled_ = enable;
    330   _mediaOpt.EnableFrameDropper(enable);
    331   return VCM_OK;
    332 }
    333 
    334 void VideoSender::SuspendBelowMinBitrate() {
    335   RTC_DCHECK(main_thread_.CalledOnValidThread());
    336   int threshold_bps;
    337   if (current_codec_.numberOfSimulcastStreams == 0) {
    338     threshold_bps = current_codec_.minBitrate * 1000;
    339   } else {
    340     threshold_bps = current_codec_.simulcastStream[0].minBitrate * 1000;
    341   }
    342   // Set the hysteresis window to be at 10% of the threshold, but at least
    343   // 10 kbps.
    344   int window_bps = std::max(threshold_bps / 10, 10000);
    345   _mediaOpt.SuspendBelowMinBitrate(threshold_bps, window_bps);
    346 }
    347 
    348 bool VideoSender::VideoSuspended() const {
    349   return _mediaOpt.IsVideoSuspended();
    350 }
    351 }  // namespace vcm
    352 }  // namespace webrtc
    353