1 /* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 12 #include <algorithm> // std::max 13 14 #include "webrtc/base/checks.h" 15 #include "webrtc/base/logging.h" 16 #include "webrtc/common_types.h" 17 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" 18 #include "webrtc/modules/video_coding/include/video_codec_interface.h" 19 #include "webrtc/modules/video_coding/encoded_frame.h" 20 #include "webrtc/modules/video_coding/utility/quality_scaler.h" 21 #include "webrtc/modules/video_coding/video_coding_impl.h" 22 #include "webrtc/system_wrappers/include/clock.h" 23 24 namespace webrtc { 25 namespace vcm { 26 27 VideoSender::VideoSender(Clock* clock, 28 EncodedImageCallback* post_encode_callback, 29 VideoEncoderRateObserver* encoder_rate_observer, 30 VCMQMSettingsCallback* qm_settings_callback) 31 : clock_(clock), 32 process_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), 33 _encoder(nullptr), 34 _encodedFrameCallback(post_encode_callback), 35 _nextFrameTypes(1, kVideoFrameDelta), 36 _mediaOpt(clock_), 37 _sendStatsCallback(nullptr), 38 _codecDataBase(encoder_rate_observer, &_encodedFrameCallback), 39 frame_dropper_enabled_(true), 40 _sendStatsTimer(1000, clock_), 41 current_codec_(), 42 qm_settings_callback_(qm_settings_callback), 43 protection_callback_(nullptr), 44 encoder_params_({0, 0, 0, 0}) { 45 // Allow VideoSender to be created on one thread but used on another, post 46 // construction. This is currently how this class is being used by at least 47 // one external project (diffractor). 48 _mediaOpt.EnableQM(qm_settings_callback_ != nullptr); 49 _mediaOpt.Reset(); 50 main_thread_.DetachFromThread(); 51 } 52 53 VideoSender::~VideoSender() {} 54 55 int32_t VideoSender::Process() { 56 int32_t returnValue = VCM_OK; 57 58 if (_sendStatsTimer.TimeUntilProcess() == 0) { 59 _sendStatsTimer.Processed(); 60 CriticalSectionScoped cs(process_crit_sect_.get()); 61 if (_sendStatsCallback != nullptr) { 62 uint32_t bitRate = _mediaOpt.SentBitRate(); 63 uint32_t frameRate = _mediaOpt.SentFrameRate(); 64 _sendStatsCallback->SendStatistics(bitRate, frameRate); 65 } 66 } 67 68 { 69 rtc::CritScope cs(¶ms_lock_); 70 // Force an encoder parameters update, so that incoming frame rate is 71 // updated even if bandwidth hasn't changed. 72 encoder_params_.input_frame_rate = _mediaOpt.InputFrameRate(); 73 } 74 75 return returnValue; 76 } 77 78 int64_t VideoSender::TimeUntilNextProcess() { 79 return _sendStatsTimer.TimeUntilProcess(); 80 } 81 82 // Register the send codec to be used. 83 int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec, 84 uint32_t numberOfCores, 85 uint32_t maxPayloadSize) { 86 RTC_DCHECK(main_thread_.CalledOnValidThread()); 87 rtc::CritScope lock(&send_crit_); 88 if (sendCodec == nullptr) { 89 return VCM_PARAMETER_ERROR; 90 } 91 92 bool ret = 93 _codecDataBase.SetSendCodec(sendCodec, numberOfCores, maxPayloadSize); 94 95 // Update encoder regardless of result to make sure that we're not holding on 96 // to a deleted instance. 97 _encoder = _codecDataBase.GetEncoder(); 98 // Cache the current codec here so they can be fetched from this thread 99 // without requiring the _sendCritSect lock. 100 current_codec_ = *sendCodec; 101 102 if (!ret) { 103 LOG(LS_ERROR) << "Failed to initialize set encoder with payload name '" 104 << sendCodec->plName << "'."; 105 return VCM_CODEC_ERROR; 106 } 107 108 int numLayers; 109 if (sendCodec->codecType == kVideoCodecVP8) { 110 numLayers = sendCodec->codecSpecific.VP8.numberOfTemporalLayers; 111 } else if (sendCodec->codecType == kVideoCodecVP9) { 112 numLayers = sendCodec->codecSpecific.VP9.numberOfTemporalLayers; 113 } else { 114 numLayers = 1; 115 } 116 117 // If we have screensharing and we have layers, we disable frame dropper. 118 bool disable_frame_dropper = 119 numLayers > 1 && sendCodec->mode == kScreensharing; 120 if (disable_frame_dropper) { 121 _mediaOpt.EnableFrameDropper(false); 122 } else if (frame_dropper_enabled_) { 123 _mediaOpt.EnableFrameDropper(true); 124 } 125 _nextFrameTypes.clear(); 126 _nextFrameTypes.resize(VCM_MAX(sendCodec->numberOfSimulcastStreams, 1), 127 kVideoFrameDelta); 128 129 _mediaOpt.SetEncodingData(sendCodec->codecType, sendCodec->maxBitrate * 1000, 130 sendCodec->startBitrate * 1000, sendCodec->width, 131 sendCodec->height, sendCodec->maxFramerate, 132 numLayers, maxPayloadSize); 133 return VCM_OK; 134 } 135 136 // Register an external decoder object. 137 // This can not be used together with external decoder callbacks. 138 void VideoSender::RegisterExternalEncoder(VideoEncoder* externalEncoder, 139 uint8_t payloadType, 140 bool internalSource /*= false*/) { 141 RTC_DCHECK(main_thread_.CalledOnValidThread()); 142 143 rtc::CritScope lock(&send_crit_); 144 145 if (externalEncoder == nullptr) { 146 bool wasSendCodec = false; 147 RTC_CHECK( 148 _codecDataBase.DeregisterExternalEncoder(payloadType, &wasSendCodec)); 149 if (wasSendCodec) { 150 // Make sure the VCM doesn't use the de-registered codec 151 _encoder = nullptr; 152 } 153 return; 154 } 155 _codecDataBase.RegisterExternalEncoder(externalEncoder, payloadType, 156 internalSource); 157 } 158 159 // Get encode bitrate 160 int VideoSender::Bitrate(unsigned int* bitrate) const { 161 RTC_DCHECK(main_thread_.CalledOnValidThread()); 162 // Since we're running on the thread that's the only thread known to modify 163 // the value of _encoder, we don't need to grab the lock here. 164 165 if (!_encoder) 166 return VCM_UNINITIALIZED; 167 *bitrate = _encoder->GetEncoderParameters().target_bitrate; 168 return 0; 169 } 170 171 // Get encode frame rate 172 int VideoSender::FrameRate(unsigned int* framerate) const { 173 RTC_DCHECK(main_thread_.CalledOnValidThread()); 174 // Since we're running on the thread that's the only thread known to modify 175 // the value of _encoder, we don't need to grab the lock here. 176 177 if (!_encoder) 178 return VCM_UNINITIALIZED; 179 180 *framerate = _encoder->GetEncoderParameters().input_frame_rate; 181 return 0; 182 } 183 184 int32_t VideoSender::SetChannelParameters(uint32_t target_bitrate, 185 uint8_t lossRate, 186 int64_t rtt) { 187 uint32_t target_rate = 188 _mediaOpt.SetTargetRates(target_bitrate, lossRate, rtt, 189 protection_callback_, qm_settings_callback_); 190 191 uint32_t input_frame_rate = _mediaOpt.InputFrameRate(); 192 193 rtc::CritScope cs(¶ms_lock_); 194 encoder_params_ = {target_rate, lossRate, rtt, input_frame_rate}; 195 196 return VCM_OK; 197 } 198 199 void VideoSender::SetEncoderParameters(EncoderParameters params) { 200 if (params.target_bitrate == 0) 201 return; 202 203 if (params.input_frame_rate == 0) { 204 // No frame rate estimate available, use default. 205 params.input_frame_rate = current_codec_.maxFramerate; 206 } 207 if (_encoder != nullptr) 208 _encoder->SetEncoderParameters(params); 209 } 210 211 int32_t VideoSender::RegisterTransportCallback( 212 VCMPacketizationCallback* transport) { 213 rtc::CritScope lock(&send_crit_); 214 _encodedFrameCallback.SetMediaOpt(&_mediaOpt); 215 _encodedFrameCallback.SetTransportCallback(transport); 216 return VCM_OK; 217 } 218 219 // Register video output information callback which will be called to deliver 220 // information about the video stream produced by the encoder, for instance the 221 // average frame rate and bit rate. 222 int32_t VideoSender::RegisterSendStatisticsCallback( 223 VCMSendStatisticsCallback* sendStats) { 224 CriticalSectionScoped cs(process_crit_sect_.get()); 225 _sendStatsCallback = sendStats; 226 return VCM_OK; 227 } 228 229 // Register a video protection callback which will be called to deliver the 230 // requested FEC rate and NACK status (on/off). 231 // Note: this callback is assumed to only be registered once and before it is 232 // used in this class. 233 int32_t VideoSender::RegisterProtectionCallback( 234 VCMProtectionCallback* protection_callback) { 235 RTC_DCHECK(protection_callback == nullptr || protection_callback_ == nullptr); 236 protection_callback_ = protection_callback; 237 return VCM_OK; 238 } 239 240 // Enable or disable a video protection method. 241 void VideoSender::SetVideoProtection(VCMVideoProtection videoProtection) { 242 rtc::CritScope lock(&send_crit_); 243 switch (videoProtection) { 244 case kProtectionNone: 245 _mediaOpt.SetProtectionMethod(media_optimization::kNone); 246 break; 247 case kProtectionNack: 248 _mediaOpt.SetProtectionMethod(media_optimization::kNack); 249 break; 250 case kProtectionNackFEC: 251 _mediaOpt.SetProtectionMethod(media_optimization::kNackFec); 252 break; 253 case kProtectionFEC: 254 _mediaOpt.SetProtectionMethod(media_optimization::kFec); 255 break; 256 } 257 } 258 // Add one raw video frame to the encoder, blocking. 259 int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame, 260 const VideoContentMetrics* contentMetrics, 261 const CodecSpecificInfo* codecSpecificInfo) { 262 EncoderParameters encoder_params; 263 { 264 rtc::CritScope lock(¶ms_lock_); 265 encoder_params = encoder_params_; 266 } 267 rtc::CritScope lock(&send_crit_); 268 if (_encoder == nullptr) 269 return VCM_UNINITIALIZED; 270 SetEncoderParameters(encoder_params); 271 // TODO(holmer): Add support for dropping frames per stream. Currently we 272 // only have one frame dropper for all streams. 273 if (_nextFrameTypes[0] == kEmptyFrame) { 274 return VCM_OK; 275 } 276 if (_mediaOpt.DropFrame()) { 277 _encoder->OnDroppedFrame(); 278 return VCM_OK; 279 } 280 _mediaOpt.UpdateContentData(contentMetrics); 281 // TODO(pbos): Make sure setting send codec is synchronized with video 282 // processing so frame size always matches. 283 if (!_codecDataBase.MatchesCurrentResolution(videoFrame.width(), 284 videoFrame.height())) { 285 LOG(LS_ERROR) << "Incoming frame doesn't match set resolution. Dropping."; 286 return VCM_PARAMETER_ERROR; 287 } 288 VideoFrame converted_frame = videoFrame; 289 if (converted_frame.native_handle() && !_encoder->SupportsNativeHandle()) { 290 // This module only supports software encoding. 291 // TODO(pbos): Offload conversion from the encoder thread. 292 converted_frame = converted_frame.ConvertNativeToI420Frame(); 293 RTC_CHECK(!converted_frame.IsZeroSize()) 294 << "Frame conversion failed, won't be able to encode frame."; 295 } 296 int32_t ret = 297 _encoder->Encode(converted_frame, codecSpecificInfo, _nextFrameTypes); 298 if (ret < 0) { 299 LOG(LS_ERROR) << "Failed to encode frame. Error code: " << ret; 300 return ret; 301 } 302 for (size_t i = 0; i < _nextFrameTypes.size(); ++i) { 303 _nextFrameTypes[i] = kVideoFrameDelta; // Default frame type. 304 } 305 if (qm_settings_callback_) 306 qm_settings_callback_->SetTargetFramerate(_encoder->GetTargetFramerate()); 307 return VCM_OK; 308 } 309 310 int32_t VideoSender::IntraFrameRequest(int stream_index) { 311 rtc::CritScope lock(&send_crit_); 312 if (stream_index < 0 || 313 static_cast<unsigned int>(stream_index) >= _nextFrameTypes.size()) { 314 return -1; 315 } 316 _nextFrameTypes[stream_index] = kVideoFrameKey; 317 if (_encoder != nullptr && _encoder->InternalSource()) { 318 // Try to request the frame if we have an external encoder with 319 // internal source since AddVideoFrame never will be called. 320 if (_encoder->RequestFrame(_nextFrameTypes) == WEBRTC_VIDEO_CODEC_OK) { 321 _nextFrameTypes[stream_index] = kVideoFrameDelta; 322 } 323 } 324 return VCM_OK; 325 } 326 327 int32_t VideoSender::EnableFrameDropper(bool enable) { 328 rtc::CritScope lock(&send_crit_); 329 frame_dropper_enabled_ = enable; 330 _mediaOpt.EnableFrameDropper(enable); 331 return VCM_OK; 332 } 333 334 void VideoSender::SuspendBelowMinBitrate() { 335 RTC_DCHECK(main_thread_.CalledOnValidThread()); 336 int threshold_bps; 337 if (current_codec_.numberOfSimulcastStreams == 0) { 338 threshold_bps = current_codec_.minBitrate * 1000; 339 } else { 340 threshold_bps = current_codec_.simulcastStream[0].minBitrate * 1000; 341 } 342 // Set the hysteresis window to be at 10% of the threshold, but at least 343 // 10 kbps. 344 int window_bps = std::max(threshold_bps / 10, 10000); 345 _mediaOpt.SuspendBelowMinBitrate(threshold_bps, window_bps); 346 } 347 348 bool VideoSender::VideoSuspended() const { 349 return _mediaOpt.IsVideoSuspended(); 350 } 351 } // namespace vcm 352 } // namespace webrtc 353