1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "webrtc/video_engine/vie_encoder.h" 12 13 #include <assert.h> 14 15 #include <algorithm> 16 17 #include "webrtc/common_video/interface/video_image.h" 18 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" 19 #include "webrtc/modules/pacing/include/paced_sender.h" 20 #include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h" 21 #include "webrtc/modules/utility/interface/process_thread.h" 22 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" 23 #include "webrtc/modules/video_coding/main/interface/video_coding.h" 24 #include "webrtc/modules/video_coding/main/interface/video_coding_defines.h" 25 #include "webrtc/modules/video_coding/main/source/encoded_frame.h" 26 #include "webrtc/system_wrappers/interface/clock.h" 27 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" 28 #include "webrtc/system_wrappers/interface/logging.h" 29 #include "webrtc/system_wrappers/interface/tick_util.h" 30 #include "webrtc/system_wrappers/interface/trace_event.h" 31 #include "webrtc/video_engine/include/vie_codec.h" 32 #include "webrtc/video_engine/include/vie_image_process.h" 33 #include "webrtc/frame_callback.h" 34 #include "webrtc/video_engine/vie_defines.h" 35 36 namespace webrtc { 37 38 // Margin on when we pause the encoder when the pacing buffer overflows relative 39 // to the configured buffer delay. 40 static const float kEncoderPausePacerMargin = 2.0f; 41 42 // Don't stop the encoder unless the delay is above this configured value. 43 static const int kMinPacingDelayMs = 200; 44 45 // Allow packets to be transmitted in up to 2 times max video bitrate if the 46 // bandwidth estimate allows it. 47 // TODO(holmer): Expose transmission start, min and max bitrates in the 48 // VideoEngine API and remove the kTransmissionMaxBitrateMultiplier. 49 static const int kTransmissionMaxBitrateMultiplier = 2; 50 51 static const float kStopPaddingThresholdMs = 2000; 52 53 std::vector<uint32_t> AllocateStreamBitrates( 54 uint32_t total_bitrate, 55 const SimulcastStream* stream_configs, 56 size_t number_of_streams) { 57 if (number_of_streams == 0) { 58 std::vector<uint32_t> stream_bitrates(1, 0); 59 stream_bitrates[0] = total_bitrate; 60 return stream_bitrates; 61 } 62 std::vector<uint32_t> stream_bitrates(number_of_streams, 0); 63 uint32_t bitrate_remainder = total_bitrate; 64 for (size_t i = 0; i < stream_bitrates.size() && bitrate_remainder > 0; ++i) { 65 if (stream_configs[i].maxBitrate * 1000 > bitrate_remainder) { 66 stream_bitrates[i] = bitrate_remainder; 67 } else { 68 stream_bitrates[i] = stream_configs[i].maxBitrate * 1000; 69 } 70 bitrate_remainder -= stream_bitrates[i]; 71 } 72 return stream_bitrates; 73 } 74 75 class QMVideoSettingsCallback : public VCMQMSettingsCallback { 76 public: 77 explicit QMVideoSettingsCallback(VideoProcessingModule* vpm); 78 79 ~QMVideoSettingsCallback(); 80 81 // Update VPM with QM (quality modes: frame size & frame rate) settings. 82 int32_t SetVideoQMSettings(const uint32_t frame_rate, 83 const uint32_t width, 84 const uint32_t height); 85 86 private: 87 VideoProcessingModule* vpm_; 88 }; 89 90 class ViEBitrateObserver : public BitrateObserver { 91 public: 92 explicit ViEBitrateObserver(ViEEncoder* owner) 93 : owner_(owner) { 94 } 95 virtual ~ViEBitrateObserver() {} 96 // Implements BitrateObserver. 97 virtual void OnNetworkChanged(const uint32_t bitrate_bps, 98 const uint8_t fraction_lost, 99 const uint32_t rtt) { 100 owner_->OnNetworkChanged(bitrate_bps, fraction_lost, rtt); 101 } 102 private: 103 ViEEncoder* owner_; 104 }; 105 106 class ViEPacedSenderCallback : public PacedSender::Callback { 107 public: 108 explicit ViEPacedSenderCallback(ViEEncoder* owner) 109 : owner_(owner) { 110 } 111 virtual ~ViEPacedSenderCallback() {} 112 virtual bool TimeToSendPacket(uint32_t ssrc, uint16_t sequence_number, 113 int64_t capture_time_ms, bool retransmission) { 114 return owner_->TimeToSendPacket(ssrc, sequence_number, capture_time_ms, 115 retransmission); 116 } 117 virtual int TimeToSendPadding(int bytes) { 118 return owner_->TimeToSendPadding(bytes); 119 } 120 private: 121 ViEEncoder* owner_; 122 }; 123 124 ViEEncoder::ViEEncoder(int32_t engine_id, 125 int32_t channel_id, 126 uint32_t number_of_cores, 127 const Config& config, 128 ProcessThread& module_process_thread, 129 BitrateController* bitrate_controller) 130 : engine_id_(engine_id), 131 channel_id_(channel_id), 132 number_of_cores_(number_of_cores), 133 vcm_(*webrtc::VideoCodingModule::Create()), 134 vpm_(*webrtc::VideoProcessingModule::Create(ViEModuleId(engine_id, 135 channel_id))), 136 callback_cs_(CriticalSectionWrapper::CreateCriticalSection()), 137 data_cs_(CriticalSectionWrapper::CreateCriticalSection()), 138 bitrate_controller_(bitrate_controller), 139 time_of_last_incoming_frame_ms_(0), 140 send_padding_(false), 141 min_transmit_bitrate_kbps_(0), 142 target_delay_ms_(0), 143 network_is_transmitting_(true), 144 encoder_paused_(false), 145 encoder_paused_and_dropped_frame_(false), 146 fec_enabled_(false), 147 nack_enabled_(false), 148 codec_observer_(NULL), 149 effect_filter_(NULL), 150 module_process_thread_(module_process_thread), 151 has_received_sli_(false), 152 picture_id_sli_(0), 153 has_received_rpsi_(false), 154 picture_id_rpsi_(0), 155 qm_callback_(NULL), 156 video_suspended_(false), 157 pre_encode_callback_(NULL) { 158 RtpRtcp::Configuration configuration; 159 configuration.id = ViEModuleId(engine_id_, channel_id_); 160 configuration.audio = false; // Video. 161 162 default_rtp_rtcp_.reset(RtpRtcp::CreateRtpRtcp(configuration)); 163 bitrate_observer_.reset(new ViEBitrateObserver(this)); 164 pacing_callback_.reset(new ViEPacedSenderCallback(this)); 165 paced_sender_.reset( 166 new PacedSender(Clock::GetRealTimeClock(), pacing_callback_.get(), 167 PacedSender::kDefaultInitialPaceKbps, 0)); 168 } 169 170 bool ViEEncoder::Init() { 171 if (vcm_.InitializeSender() != 0) { 172 return false; 173 } 174 vpm_.EnableTemporalDecimation(true); 175 176 // Enable/disable content analysis: off by default for now. 177 vpm_.EnableContentAnalysis(false); 178 179 if (module_process_thread_.RegisterModule(&vcm_) != 0 || 180 module_process_thread_.RegisterModule(default_rtp_rtcp_.get()) != 0 || 181 module_process_thread_.RegisterModule(paced_sender_.get()) != 0) { 182 return false; 183 } 184 if (qm_callback_) { 185 delete qm_callback_; 186 } 187 qm_callback_ = new QMVideoSettingsCallback(&vpm_); 188 189 #ifdef VIDEOCODEC_VP8 190 VideoCodecType codec_type = webrtc::kVideoCodecVP8; 191 #else 192 VideoCodecType codec_type = webrtc::kVideoCodecI420; 193 #endif 194 195 VideoCodec video_codec; 196 if (vcm_.Codec(codec_type, &video_codec) != VCM_OK) { 197 return false; 198 } 199 { 200 CriticalSectionScoped cs(data_cs_.get()); 201 send_padding_ = video_codec.numberOfSimulcastStreams > 1; 202 } 203 if (vcm_.RegisterSendCodec(&video_codec, number_of_cores_, 204 default_rtp_rtcp_->MaxDataPayloadLength()) != 0) { 205 return false; 206 } 207 if (default_rtp_rtcp_->RegisterSendPayload(video_codec) != 0) { 208 return false; 209 } 210 if (vcm_.RegisterTransportCallback(this) != 0) { 211 return false; 212 } 213 if (vcm_.RegisterSendStatisticsCallback(this) != 0) { 214 return false; 215 } 216 if (vcm_.RegisterVideoQMCallback(qm_callback_) != 0) { 217 return false; 218 } 219 return true; 220 } 221 222 ViEEncoder::~ViEEncoder() { 223 if (bitrate_controller_) { 224 bitrate_controller_->RemoveBitrateObserver(bitrate_observer_.get()); 225 } 226 module_process_thread_.DeRegisterModule(&vcm_); 227 module_process_thread_.DeRegisterModule(&vpm_); 228 module_process_thread_.DeRegisterModule(default_rtp_rtcp_.get()); 229 module_process_thread_.DeRegisterModule(paced_sender_.get()); 230 VideoCodingModule::Destroy(&vcm_); 231 VideoProcessingModule::Destroy(&vpm_); 232 delete qm_callback_; 233 } 234 235 int ViEEncoder::Owner() const { 236 return channel_id_; 237 } 238 239 void ViEEncoder::SetNetworkTransmissionState(bool is_transmitting) { 240 { 241 CriticalSectionScoped cs(data_cs_.get()); 242 network_is_transmitting_ = is_transmitting; 243 } 244 if (is_transmitting) { 245 paced_sender_->Resume(); 246 } else { 247 paced_sender_->Pause(); 248 } 249 } 250 251 void ViEEncoder::Pause() { 252 CriticalSectionScoped cs(data_cs_.get()); 253 encoder_paused_ = true; 254 } 255 256 void ViEEncoder::Restart() { 257 CriticalSectionScoped cs(data_cs_.get()); 258 encoder_paused_ = false; 259 } 260 261 uint8_t ViEEncoder::NumberOfCodecs() { 262 return vcm_.NumberOfCodecs(); 263 } 264 265 int32_t ViEEncoder::GetCodec(uint8_t list_index, VideoCodec* video_codec) { 266 if (vcm_.Codec(list_index, video_codec) != 0) { 267 return -1; 268 } 269 return 0; 270 } 271 272 int32_t ViEEncoder::RegisterExternalEncoder(webrtc::VideoEncoder* encoder, 273 uint8_t pl_type, 274 bool internal_source) { 275 if (encoder == NULL) 276 return -1; 277 278 if (vcm_.RegisterExternalEncoder(encoder, pl_type, internal_source) != 279 VCM_OK) { 280 return -1; 281 } 282 return 0; 283 } 284 285 int32_t ViEEncoder::DeRegisterExternalEncoder(uint8_t pl_type) { 286 webrtc::VideoCodec current_send_codec; 287 if (vcm_.SendCodec(¤t_send_codec) == VCM_OK) { 288 uint32_t current_bitrate_bps = 0; 289 if (vcm_.Bitrate(¤t_bitrate_bps) != 0) { 290 LOG(LS_WARNING) << "Failed to get the current encoder target bitrate."; 291 } 292 current_send_codec.startBitrate = (current_bitrate_bps + 500) / 1000; 293 } 294 295 if (vcm_.RegisterExternalEncoder(NULL, pl_type) != VCM_OK) { 296 return -1; 297 } 298 299 // If the external encoder is the current send codec, use vcm internal 300 // encoder. 301 if (current_send_codec.plType == pl_type) { 302 uint16_t max_data_payload_length = 303 default_rtp_rtcp_->MaxDataPayloadLength(); 304 { 305 CriticalSectionScoped cs(data_cs_.get()); 306 send_padding_ = current_send_codec.numberOfSimulcastStreams > 1; 307 } 308 // TODO(mflodman): Unfortunately the VideoCodec that VCM has cached a 309 // raw pointer to an |extra_options| that's long gone. Clearing it here is 310 // a hack to prevent the following code from crashing. This should be fixed 311 // for realz. https://code.google.com/p/chromium/issues/detail?id=348222 312 current_send_codec.extra_options = NULL; 313 if (vcm_.RegisterSendCodec(¤t_send_codec, number_of_cores_, 314 max_data_payload_length) != VCM_OK) { 315 LOG(LS_INFO) << "De-registered the currently used external encoder (" 316 << static_cast<int>(pl_type) << ") and therefore tried to " 317 << "register the corresponding internal encoder, but none " 318 << "was supported."; 319 } 320 } 321 return 0; 322 } 323 324 int32_t ViEEncoder::SetEncoder(const webrtc::VideoCodec& video_codec) { 325 // Setting target width and height for VPM. 326 if (vpm_.SetTargetResolution(video_codec.width, video_codec.height, 327 video_codec.maxFramerate) != VPM_OK) { 328 return -1; 329 } 330 331 if (default_rtp_rtcp_->RegisterSendPayload(video_codec) != 0) { 332 return -1; 333 } 334 // Convert from kbps to bps. 335 std::vector<uint32_t> stream_bitrates = AllocateStreamBitrates( 336 video_codec.startBitrate * 1000, 337 video_codec.simulcastStream, 338 video_codec.numberOfSimulcastStreams); 339 default_rtp_rtcp_->SetTargetSendBitrate(stream_bitrates); 340 341 uint16_t max_data_payload_length = 342 default_rtp_rtcp_->MaxDataPayloadLength(); 343 344 { 345 CriticalSectionScoped cs(data_cs_.get()); 346 send_padding_ = video_codec.numberOfSimulcastStreams > 1; 347 } 348 if (vcm_.RegisterSendCodec(&video_codec, number_of_cores_, 349 max_data_payload_length) != VCM_OK) { 350 return -1; 351 } 352 353 // Set this module as sending right away, let the slave module in the channel 354 // start and stop sending. 355 if (default_rtp_rtcp_->SetSendingStatus(true) != 0) { 356 return -1; 357 } 358 359 bitrate_controller_->SetBitrateObserver(bitrate_observer_.get(), 360 video_codec.startBitrate * 1000, 361 video_codec.minBitrate * 1000, 362 kTransmissionMaxBitrateMultiplier * 363 video_codec.maxBitrate * 1000); 364 365 CriticalSectionScoped crit(data_cs_.get()); 366 int pad_up_to_bitrate_kbps = video_codec.startBitrate; 367 if (pad_up_to_bitrate_kbps < min_transmit_bitrate_kbps_) 368 pad_up_to_bitrate_kbps = min_transmit_bitrate_kbps_; 369 370 paced_sender_->UpdateBitrate( 371 PacedSender::kDefaultPaceMultiplier * video_codec.startBitrate, 372 pad_up_to_bitrate_kbps); 373 374 return 0; 375 } 376 377 int32_t ViEEncoder::GetEncoder(VideoCodec* video_codec) { 378 if (vcm_.SendCodec(video_codec) != 0) { 379 return -1; 380 } 381 return 0; 382 } 383 384 int32_t ViEEncoder::GetCodecConfigParameters( 385 unsigned char config_parameters[kConfigParameterSize], 386 unsigned char& config_parameters_size) { 387 int32_t num_parameters = 388 vcm_.CodecConfigParameters(config_parameters, kConfigParameterSize); 389 if (num_parameters <= 0) { 390 config_parameters_size = 0; 391 return -1; 392 } 393 config_parameters_size = static_cast<unsigned char>(num_parameters); 394 return 0; 395 } 396 397 int32_t ViEEncoder::ScaleInputImage(bool enable) { 398 VideoFrameResampling resampling_mode = kFastRescaling; 399 // TODO(mflodman) What? 400 if (enable) { 401 // kInterpolation is currently not supported. 402 LOG_F(LS_ERROR) << "Not supported."; 403 return -1; 404 } 405 vpm_.SetInputFrameResampleMode(resampling_mode); 406 407 return 0; 408 } 409 410 bool ViEEncoder::TimeToSendPacket(uint32_t ssrc, 411 uint16_t sequence_number, 412 int64_t capture_time_ms, 413 bool retransmission) { 414 return default_rtp_rtcp_->TimeToSendPacket(ssrc, sequence_number, 415 capture_time_ms, retransmission); 416 } 417 418 int ViEEncoder::TimeToSendPadding(int bytes) { 419 bool send_padding; 420 { 421 CriticalSectionScoped cs(data_cs_.get()); 422 send_padding = 423 send_padding_ || video_suspended_ || min_transmit_bitrate_kbps_ > 0; 424 } 425 if (send_padding) { 426 return default_rtp_rtcp_->TimeToSendPadding(bytes); 427 } 428 return 0; 429 } 430 431 bool ViEEncoder::EncoderPaused() const { 432 // Pause video if paused by caller or as long as the network is down or the 433 // pacer queue has grown too large in buffered mode. 434 if (encoder_paused_) { 435 return true; 436 } 437 if (target_delay_ms_ > 0) { 438 // Buffered mode. 439 // TODO(pwestin): Workaround until nack is configured as a time and not 440 // number of packets. 441 return paced_sender_->QueueInMs() >= 442 std::max(static_cast<int>(target_delay_ms_ * kEncoderPausePacerMargin), 443 kMinPacingDelayMs); 444 } 445 return !network_is_transmitting_; 446 } 447 448 RtpRtcp* ViEEncoder::SendRtpRtcpModule() { 449 return default_rtp_rtcp_.get(); 450 } 451 452 void ViEEncoder::DeliverFrame(int id, 453 I420VideoFrame* video_frame, 454 int num_csrcs, 455 const uint32_t CSRC[kRtpCsrcSize]) { 456 if (default_rtp_rtcp_->SendingMedia() == false) { 457 // We've paused or we have no channels attached, don't encode. 458 return; 459 } 460 { 461 CriticalSectionScoped cs(data_cs_.get()); 462 time_of_last_incoming_frame_ms_ = TickTime::MillisecondTimestamp(); 463 if (EncoderPaused()) { 464 if (!encoder_paused_and_dropped_frame_) { 465 TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this); 466 } 467 encoder_paused_and_dropped_frame_ = true; 468 return; 469 } 470 if (encoder_paused_and_dropped_frame_) { 471 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); 472 } 473 encoder_paused_and_dropped_frame_ = false; 474 } 475 476 // Convert render time, in ms, to RTP timestamp. 477 const int kMsToRtpTimestamp = 90; 478 const uint32_t time_stamp = 479 kMsToRtpTimestamp * 480 static_cast<uint32_t>(video_frame->render_time_ms()); 481 482 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame->render_time_ms(), 483 "Encode"); 484 video_frame->set_timestamp(time_stamp); 485 486 // Make sure the CSRC list is correct. 487 if (num_csrcs > 0) { 488 uint32_t tempCSRC[kRtpCsrcSize]; 489 for (int i = 0; i < num_csrcs; i++) { 490 if (CSRC[i] == 1) { 491 tempCSRC[i] = default_rtp_rtcp_->SSRC(); 492 } else { 493 tempCSRC[i] = CSRC[i]; 494 } 495 } 496 default_rtp_rtcp_->SetCSRCs(tempCSRC, (uint8_t) num_csrcs); 497 } 498 499 I420VideoFrame* decimated_frame = NULL; 500 // TODO(wuchengli): support texture frames. 501 if (video_frame->native_handle() == NULL) { 502 { 503 CriticalSectionScoped cs(callback_cs_.get()); 504 if (effect_filter_) { 505 unsigned int length = 506 CalcBufferSize(kI420, video_frame->width(), video_frame->height()); 507 scoped_ptr<uint8_t[]> video_buffer(new uint8_t[length]); 508 ExtractBuffer(*video_frame, length, video_buffer.get()); 509 effect_filter_->Transform(length, 510 video_buffer.get(), 511 video_frame->ntp_time_ms(), 512 video_frame->timestamp(), 513 video_frame->width(), 514 video_frame->height()); 515 } 516 } 517 518 // Pass frame via preprocessor. 519 const int ret = vpm_.PreprocessFrame(*video_frame, &decimated_frame); 520 if (ret == 1) { 521 // Drop this frame. 522 return; 523 } 524 if (ret != VPM_OK) { 525 return; 526 } 527 } 528 // If the frame was not resampled or scaled => use original. 529 if (decimated_frame == NULL) { 530 decimated_frame = video_frame; 531 } 532 533 { 534 CriticalSectionScoped cs(callback_cs_.get()); 535 if (pre_encode_callback_) 536 pre_encode_callback_->FrameCallback(decimated_frame); 537 } 538 539 if (video_frame->native_handle() != NULL) { 540 // TODO(wuchengli): add texture support. http://crbug.com/362437 541 return; 542 } 543 544 #ifdef VIDEOCODEC_VP8 545 if (vcm_.SendCodec() == webrtc::kVideoCodecVP8) { 546 webrtc::CodecSpecificInfo codec_specific_info; 547 codec_specific_info.codecType = webrtc::kVideoCodecVP8; 548 { 549 CriticalSectionScoped cs(data_cs_.get()); 550 codec_specific_info.codecSpecific.VP8.hasReceivedRPSI = 551 has_received_rpsi_; 552 codec_specific_info.codecSpecific.VP8.hasReceivedSLI = 553 has_received_sli_; 554 codec_specific_info.codecSpecific.VP8.pictureIdRPSI = 555 picture_id_rpsi_; 556 codec_specific_info.codecSpecific.VP8.pictureIdSLI = 557 picture_id_sli_; 558 has_received_sli_ = false; 559 has_received_rpsi_ = false; 560 } 561 562 vcm_.AddVideoFrame(*decimated_frame, vpm_.ContentMetrics(), 563 &codec_specific_info); 564 return; 565 } 566 #endif 567 vcm_.AddVideoFrame(*decimated_frame); 568 } 569 570 void ViEEncoder::DelayChanged(int id, int frame_delay) { 571 default_rtp_rtcp_->SetCameraDelay(frame_delay); 572 } 573 574 int ViEEncoder::GetPreferedFrameSettings(int* width, 575 int* height, 576 int* frame_rate) { 577 webrtc::VideoCodec video_codec; 578 memset(&video_codec, 0, sizeof(video_codec)); 579 if (vcm_.SendCodec(&video_codec) != VCM_OK) { 580 return -1; 581 } 582 583 *width = video_codec.width; 584 *height = video_codec.height; 585 *frame_rate = video_codec.maxFramerate; 586 return 0; 587 } 588 589 int ViEEncoder::SendKeyFrame() { 590 return vcm_.IntraFrameRequest(0); 591 } 592 593 int32_t ViEEncoder::SendCodecStatistics( 594 uint32_t* num_key_frames, uint32_t* num_delta_frames) { 595 webrtc::VCMFrameCount sent_frames; 596 if (vcm_.SentFrameCount(sent_frames) != VCM_OK) { 597 return -1; 598 } 599 *num_key_frames = sent_frames.numKeyFrames; 600 *num_delta_frames = sent_frames.numDeltaFrames; 601 return 0; 602 } 603 604 int32_t ViEEncoder::PacerQueuingDelayMs() const { 605 return paced_sender_->QueueInMs(); 606 } 607 608 int ViEEncoder::CodecTargetBitrate(uint32_t* bitrate) const { 609 if (vcm_.Bitrate(bitrate) != 0) 610 return -1; 611 return 0; 612 } 613 614 int32_t ViEEncoder::UpdateProtectionMethod(bool enable_nack) { 615 bool fec_enabled = false; 616 uint8_t dummy_ptype_red = 0; 617 uint8_t dummy_ptypeFEC = 0; 618 619 // Updated protection method to VCM to get correct packetization sizes. 620 // FEC has larger overhead than NACK -> set FEC if used. 621 int32_t error = default_rtp_rtcp_->GenericFECStatus(fec_enabled, 622 dummy_ptype_red, 623 dummy_ptypeFEC); 624 if (error) { 625 return -1; 626 } 627 if (fec_enabled_ == fec_enabled && nack_enabled_ == enable_nack) { 628 // No change needed, we're already in correct state. 629 return 0; 630 } 631 fec_enabled_ = fec_enabled; 632 nack_enabled_ = enable_nack; 633 634 // Set Video Protection for VCM. 635 if (fec_enabled && nack_enabled_) { 636 vcm_.SetVideoProtection(webrtc::kProtectionNackFEC, true); 637 } else { 638 vcm_.SetVideoProtection(webrtc::kProtectionFEC, fec_enabled_); 639 vcm_.SetVideoProtection(webrtc::kProtectionNackSender, nack_enabled_); 640 vcm_.SetVideoProtection(webrtc::kProtectionNackFEC, false); 641 } 642 643 if (fec_enabled_ || nack_enabled_) { 644 vcm_.RegisterProtectionCallback(this); 645 // The send codec must be registered to set correct MTU. 646 webrtc::VideoCodec codec; 647 if (vcm_.SendCodec(&codec) == 0) { 648 uint16_t max_pay_load = default_rtp_rtcp_->MaxDataPayloadLength(); 649 uint32_t current_bitrate_bps = 0; 650 if (vcm_.Bitrate(¤t_bitrate_bps) != 0) { 651 LOG_F(LS_WARNING) << 652 "Failed to get the current encoder target bitrate."; 653 } 654 // Convert to start bitrate in kbps. 655 codec.startBitrate = (current_bitrate_bps + 500) / 1000; 656 if (vcm_.RegisterSendCodec(&codec, number_of_cores_, max_pay_load) != 0) { 657 return -1; 658 } 659 } 660 return 0; 661 } else { 662 // FEC and NACK are disabled. 663 vcm_.RegisterProtectionCallback(NULL); 664 } 665 return 0; 666 } 667 668 void ViEEncoder::SetSenderBufferingMode(int target_delay_ms) { 669 { 670 CriticalSectionScoped cs(data_cs_.get()); 671 target_delay_ms_ = target_delay_ms; 672 } 673 if (target_delay_ms > 0) { 674 // Disable external frame-droppers. 675 vcm_.EnableFrameDropper(false); 676 vpm_.EnableTemporalDecimation(false); 677 // We don't put any limits on the pacer queue when running in buffered mode 678 // since the encoder will be paused if the queue grow too large. 679 paced_sender_->set_max_queue_length_ms(-1); 680 } else { 681 // Real-time mode - enable frame droppers. 682 vpm_.EnableTemporalDecimation(true); 683 vcm_.EnableFrameDropper(true); 684 paced_sender_->set_max_queue_length_ms( 685 PacedSender::kDefaultMaxQueueLengthMs); 686 } 687 } 688 689 int32_t ViEEncoder::SendData( 690 const FrameType frame_type, 691 const uint8_t payload_type, 692 const uint32_t time_stamp, 693 int64_t capture_time_ms, 694 const uint8_t* payload_data, 695 const uint32_t payload_size, 696 const webrtc::RTPFragmentationHeader& fragmentation_header, 697 const RTPVideoHeader* rtp_video_hdr) { 698 // New encoded data, hand over to the rtp module. 699 return default_rtp_rtcp_->SendOutgoingData(frame_type, 700 payload_type, 701 time_stamp, 702 capture_time_ms, 703 payload_data, 704 payload_size, 705 &fragmentation_header, 706 rtp_video_hdr); 707 } 708 709 int32_t ViEEncoder::ProtectionRequest( 710 const FecProtectionParams* delta_fec_params, 711 const FecProtectionParams* key_fec_params, 712 uint32_t* sent_video_rate_bps, 713 uint32_t* sent_nack_rate_bps, 714 uint32_t* sent_fec_rate_bps) { 715 default_rtp_rtcp_->SetFecParameters(delta_fec_params, key_fec_params); 716 default_rtp_rtcp_->BitrateSent(NULL, sent_video_rate_bps, sent_fec_rate_bps, 717 sent_nack_rate_bps); 718 return 0; 719 } 720 721 int32_t ViEEncoder::SendStatistics(const uint32_t bit_rate, 722 const uint32_t frame_rate) { 723 CriticalSectionScoped cs(callback_cs_.get()); 724 if (codec_observer_) { 725 codec_observer_->OutgoingRate(channel_id_, frame_rate, bit_rate); 726 } 727 return 0; 728 } 729 730 int32_t ViEEncoder::RegisterCodecObserver(ViEEncoderObserver* observer) { 731 CriticalSectionScoped cs(callback_cs_.get()); 732 if (observer && codec_observer_) { 733 LOG_F(LS_ERROR) << "Observer already set."; 734 return -1; 735 } 736 codec_observer_ = observer; 737 return 0; 738 } 739 740 void ViEEncoder::OnReceivedSLI(uint32_t /*ssrc*/, 741 uint8_t picture_id) { 742 CriticalSectionScoped cs(data_cs_.get()); 743 picture_id_sli_ = picture_id; 744 has_received_sli_ = true; 745 } 746 747 void ViEEncoder::OnReceivedRPSI(uint32_t /*ssrc*/, 748 uint64_t picture_id) { 749 CriticalSectionScoped cs(data_cs_.get()); 750 picture_id_rpsi_ = picture_id; 751 has_received_rpsi_ = true; 752 } 753 754 void ViEEncoder::OnReceivedIntraFrameRequest(uint32_t ssrc) { 755 // Key frame request from remote side, signal to VCM. 756 TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); 757 758 int idx = 0; 759 { 760 CriticalSectionScoped cs(data_cs_.get()); 761 std::map<unsigned int, int>::iterator stream_it = ssrc_streams_.find(ssrc); 762 if (stream_it == ssrc_streams_.end()) { 763 LOG_F(LS_WARNING) << "ssrc not found: " << ssrc << ", map size " 764 << ssrc_streams_.size(); 765 return; 766 } 767 std::map<unsigned int, int64_t>::iterator time_it = 768 time_last_intra_request_ms_.find(ssrc); 769 if (time_it == time_last_intra_request_ms_.end()) { 770 time_last_intra_request_ms_[ssrc] = 0; 771 } 772 773 int64_t now = TickTime::MillisecondTimestamp(); 774 if (time_last_intra_request_ms_[ssrc] + kViEMinKeyRequestIntervalMs > now) { 775 return; 776 } 777 time_last_intra_request_ms_[ssrc] = now; 778 idx = stream_it->second; 779 } 780 // Release the critsect before triggering key frame. 781 vcm_.IntraFrameRequest(idx); 782 } 783 784 void ViEEncoder::OnLocalSsrcChanged(uint32_t old_ssrc, uint32_t new_ssrc) { 785 CriticalSectionScoped cs(data_cs_.get()); 786 std::map<unsigned int, int>::iterator it = ssrc_streams_.find(old_ssrc); 787 if (it == ssrc_streams_.end()) { 788 return; 789 } 790 791 ssrc_streams_[new_ssrc] = it->second; 792 ssrc_streams_.erase(it); 793 794 std::map<unsigned int, int64_t>::iterator time_it = 795 time_last_intra_request_ms_.find(old_ssrc); 796 int64_t last_intra_request_ms = 0; 797 if (time_it != time_last_intra_request_ms_.end()) { 798 last_intra_request_ms = time_it->second; 799 time_last_intra_request_ms_.erase(time_it); 800 } 801 time_last_intra_request_ms_[new_ssrc] = last_intra_request_ms; 802 } 803 804 bool ViEEncoder::SetSsrcs(const std::list<unsigned int>& ssrcs) { 805 VideoCodec codec; 806 if (vcm_.SendCodec(&codec) != 0) 807 return false; 808 809 if (codec.numberOfSimulcastStreams > 0 && 810 ssrcs.size() != codec.numberOfSimulcastStreams) { 811 return false; 812 } 813 814 CriticalSectionScoped cs(data_cs_.get()); 815 ssrc_streams_.clear(); 816 time_last_intra_request_ms_.clear(); 817 int idx = 0; 818 for (std::list<unsigned int>::const_iterator it = ssrcs.begin(); 819 it != ssrcs.end(); ++it, ++idx) { 820 unsigned int ssrc = *it; 821 ssrc_streams_[ssrc] = idx; 822 } 823 return true; 824 } 825 826 void ViEEncoder::SetMinTransmitBitrate(int min_transmit_bitrate_kbps) { 827 assert(min_transmit_bitrate_kbps >= 0); 828 CriticalSectionScoped crit(data_cs_.get()); 829 min_transmit_bitrate_kbps_ = min_transmit_bitrate_kbps; 830 } 831 832 // Called from ViEBitrateObserver. 833 void ViEEncoder::OnNetworkChanged(const uint32_t bitrate_bps, 834 const uint8_t fraction_lost, 835 const uint32_t round_trip_time_ms) { 836 LOG(LS_VERBOSE) << "OnNetworkChanged, bitrate" << bitrate_bps 837 << " packet loss " << fraction_lost 838 << " rtt " << round_trip_time_ms; 839 vcm_.SetChannelParameters(bitrate_bps, fraction_lost, round_trip_time_ms); 840 bool video_is_suspended = vcm_.VideoSuspended(); 841 int bitrate_kbps = bitrate_bps / 1000; 842 VideoCodec send_codec; 843 if (vcm_.SendCodec(&send_codec) != 0) { 844 return; 845 } 846 SimulcastStream* stream_configs = send_codec.simulcastStream; 847 // Allocate the bandwidth between the streams. 848 std::vector<uint32_t> stream_bitrates = AllocateStreamBitrates( 849 bitrate_bps, 850 stream_configs, 851 send_codec.numberOfSimulcastStreams); 852 // Find the max amount of padding we can allow ourselves to send at this 853 // point, based on which streams are currently active and what our current 854 // available bandwidth is. 855 int pad_up_to_bitrate_kbps = 0; 856 if (send_codec.numberOfSimulcastStreams == 0) { 857 pad_up_to_bitrate_kbps = send_codec.minBitrate; 858 } else { 859 pad_up_to_bitrate_kbps = 860 stream_configs[send_codec.numberOfSimulcastStreams - 1].minBitrate; 861 for (int i = 0; i < send_codec.numberOfSimulcastStreams - 1; ++i) { 862 pad_up_to_bitrate_kbps += stream_configs[i].targetBitrate; 863 } 864 } 865 866 // Disable padding if only sending one stream and video isn't suspended and 867 // min-transmit bitrate isn't used (applied later). 868 if (!video_is_suspended && send_codec.numberOfSimulcastStreams <= 1) 869 pad_up_to_bitrate_kbps = 0; 870 871 { 872 CriticalSectionScoped cs(data_cs_.get()); 873 // The amount of padding should decay to zero if no frames are being 874 // captured unless a min-transmit bitrate is used. 875 int64_t now_ms = TickTime::MillisecondTimestamp(); 876 if (now_ms - time_of_last_incoming_frame_ms_ > kStopPaddingThresholdMs) 877 pad_up_to_bitrate_kbps = 0; 878 879 // Pad up to min bitrate. 880 if (pad_up_to_bitrate_kbps < min_transmit_bitrate_kbps_) 881 pad_up_to_bitrate_kbps = min_transmit_bitrate_kbps_; 882 883 // Padding may never exceed bitrate estimate. 884 if (pad_up_to_bitrate_kbps > bitrate_kbps) 885 pad_up_to_bitrate_kbps = bitrate_kbps; 886 887 paced_sender_->UpdateBitrate( 888 PacedSender::kDefaultPaceMultiplier * bitrate_kbps, 889 pad_up_to_bitrate_kbps); 890 default_rtp_rtcp_->SetTargetSendBitrate(stream_bitrates); 891 if (video_suspended_ == video_is_suspended) 892 return; 893 video_suspended_ = video_is_suspended; 894 } 895 896 // Video suspend-state changed, inform codec observer. 897 CriticalSectionScoped crit(callback_cs_.get()); 898 if (codec_observer_) { 899 LOG(LS_INFO) << "Video suspended " << video_is_suspended 900 << " for channel " << channel_id_; 901 codec_observer_->SuspendChange(channel_id_, video_is_suspended); 902 } 903 } 904 905 PacedSender* ViEEncoder::GetPacedSender() { 906 return paced_sender_.get(); 907 } 908 909 int32_t ViEEncoder::RegisterEffectFilter(ViEEffectFilter* effect_filter) { 910 CriticalSectionScoped cs(callback_cs_.get()); 911 if (effect_filter != NULL && effect_filter_ != NULL) { 912 LOG_F(LS_ERROR) << "Filter already set."; 913 return -1; 914 } 915 effect_filter_ = effect_filter; 916 return 0; 917 } 918 919 int ViEEncoder::StartDebugRecording(const char* fileNameUTF8) { 920 return vcm_.StartDebugRecording(fileNameUTF8); 921 } 922 923 int ViEEncoder::StopDebugRecording() { 924 return vcm_.StopDebugRecording(); 925 } 926 927 void ViEEncoder::SuspendBelowMinBitrate() { 928 vcm_.SuspendBelowMinBitrate(); 929 bitrate_controller_->EnforceMinBitrate(false); 930 } 931 932 void ViEEncoder::RegisterPreEncodeCallback( 933 I420FrameCallback* pre_encode_callback) { 934 CriticalSectionScoped cs(callback_cs_.get()); 935 pre_encode_callback_ = pre_encode_callback; 936 } 937 938 void ViEEncoder::DeRegisterPreEncodeCallback() { 939 CriticalSectionScoped cs(callback_cs_.get()); 940 pre_encode_callback_ = NULL; 941 } 942 943 void ViEEncoder::RegisterPostEncodeImageCallback( 944 EncodedImageCallback* post_encode_callback) { 945 vcm_.RegisterPostEncodeImageCallback(post_encode_callback); 946 } 947 948 void ViEEncoder::DeRegisterPostEncodeImageCallback() { 949 vcm_.RegisterPostEncodeImageCallback(NULL); 950 } 951 952 QMVideoSettingsCallback::QMVideoSettingsCallback(VideoProcessingModule* vpm) 953 : vpm_(vpm) { 954 } 955 956 QMVideoSettingsCallback::~QMVideoSettingsCallback() { 957 } 958 959 int32_t QMVideoSettingsCallback::SetVideoQMSettings( 960 const uint32_t frame_rate, 961 const uint32_t width, 962 const uint32_t height) { 963 return vpm_->SetTargetResolution(width, height, frame_rate); 964 } 965 966 } // namespace webrtc 967
