1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "webrtc/video_engine/vie_encoder.h" 12 13 #include <assert.h> 14 15 #include <algorithm> 16 17 #include "webrtc/common_video/interface/video_image.h" 18 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" 19 #include "webrtc/modules/pacing/include/paced_sender.h" 20 #include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h" 21 #include "webrtc/modules/utility/interface/process_thread.h" 22 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" 23 #include "webrtc/modules/video_coding/main/interface/video_coding.h" 24 #include "webrtc/modules/video_coding/main/interface/video_coding_defines.h" 25 #include "webrtc/modules/video_coding/main/source/encoded_frame.h" 26 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" 27 #include "webrtc/system_wrappers/interface/logging.h" 28 #include "webrtc/system_wrappers/interface/tick_util.h" 29 #include "webrtc/system_wrappers/interface/trace_event.h" 30 #include "webrtc/video_engine/include/vie_codec.h" 31 #include "webrtc/video_engine/include/vie_image_process.h" 32 #include "webrtc/frame_callback.h" 33 #include "webrtc/video_engine/vie_defines.h" 34 35 namespace webrtc { 36 37 // Pace in kbits/s until we receive first estimate. 38 static const int kInitialPace = 2000; 39 40 // Pacing-rate relative to our target send rate. 41 // Multiplicative factor that is applied to the target bitrate to calculate the 42 // number of bytes that can be transmitted per interval. 43 // Increasing this factor will result in lower delays in cases of bitrate 44 // overshoots from the encoder. 45 static const float kPaceMultiplier = 2.5f; 46 47 // Margin on when we pause the encoder when the pacing buffer overflows relative 48 // to the configured buffer delay. 49 static const float kEncoderPausePacerMargin = 2.0f; 50 51 // Don't stop the encoder unless the delay is above this configured value. 52 static const int kMinPacingDelayMs = 200; 53 54 // Allow packets to be transmitted in up to 2 times max video bitrate if the 55 // bandwidth estimate allows it. 56 // TODO(holmer): Expose transmission start, min and max bitrates in the 57 // VideoEngine API and remove the kTransmissionMaxBitrateMultiplier. 58 static const int kTransmissionMaxBitrateMultiplier = 2; 59 60 static const float kStopPaddingThresholdMs = 2000; 61 62 std::vector<uint32_t> AllocateStreamBitrates( 63 uint32_t total_bitrate, 64 const SimulcastStream* stream_configs, 65 size_t number_of_streams) { 66 if (number_of_streams == 0) { 67 std::vector<uint32_t> stream_bitrates(1, 0); 68 stream_bitrates[0] = total_bitrate; 69 return stream_bitrates; 70 } 71 std::vector<uint32_t> stream_bitrates(number_of_streams, 0); 72 uint32_t bitrate_remainder = total_bitrate; 73 for (size_t i = 0; i < stream_bitrates.size() && bitrate_remainder > 0; ++i) { 74 if (stream_configs[i].maxBitrate * 1000 > bitrate_remainder) { 75 stream_bitrates[i] = bitrate_remainder; 76 } else { 77 stream_bitrates[i] = stream_configs[i].maxBitrate * 1000; 78 } 79 bitrate_remainder -= stream_bitrates[i]; 80 } 81 return stream_bitrates; 82 } 83 84 class QMVideoSettingsCallback : public VCMQMSettingsCallback { 85 public: 86 explicit QMVideoSettingsCallback(VideoProcessingModule* vpm); 87 88 ~QMVideoSettingsCallback(); 89 90 // Update VPM with QM (quality modes: frame size & frame rate) settings. 91 int32_t SetVideoQMSettings(const uint32_t frame_rate, 92 const uint32_t width, 93 const uint32_t height); 94 95 private: 96 VideoProcessingModule* vpm_; 97 }; 98 99 class ViEBitrateObserver : public BitrateObserver { 100 public: 101 explicit ViEBitrateObserver(ViEEncoder* owner) 102 : owner_(owner) { 103 } 104 virtual ~ViEBitrateObserver() {} 105 // Implements BitrateObserver. 106 virtual void OnNetworkChanged(const uint32_t bitrate_bps, 107 const uint8_t fraction_lost, 108 const uint32_t rtt) { 109 owner_->OnNetworkChanged(bitrate_bps, fraction_lost, rtt); 110 } 111 private: 112 ViEEncoder* owner_; 113 }; 114 115 class ViEPacedSenderCallback : public PacedSender::Callback { 116 public: 117 explicit ViEPacedSenderCallback(ViEEncoder* owner) 118 : owner_(owner) { 119 } 120 virtual ~ViEPacedSenderCallback() {} 121 virtual bool TimeToSendPacket(uint32_t ssrc, uint16_t sequence_number, 122 int64_t capture_time_ms, bool retransmission) { 123 return owner_->TimeToSendPacket(ssrc, sequence_number, capture_time_ms, 124 retransmission); 125 } 126 virtual int TimeToSendPadding(int bytes) { 127 return owner_->TimeToSendPadding(bytes); 128 } 129 private: 130 ViEEncoder* owner_; 131 }; 132 133 ViEEncoder::ViEEncoder(int32_t engine_id, 134 int32_t channel_id, 135 uint32_t number_of_cores, 136 const Config& config, 137 ProcessThread& module_process_thread, 138 BitrateController* bitrate_controller) 139 : engine_id_(engine_id), 140 channel_id_(channel_id), 141 number_of_cores_(number_of_cores), 142 vcm_(*webrtc::VideoCodingModule::Create()), 143 vpm_(*webrtc::VideoProcessingModule::Create(ViEModuleId(engine_id, 144 channel_id))), 145 callback_cs_(CriticalSectionWrapper::CreateCriticalSection()), 146 data_cs_(CriticalSectionWrapper::CreateCriticalSection()), 147 bitrate_controller_(bitrate_controller), 148 time_of_last_incoming_frame_ms_(0), 149 send_padding_(false), 150 min_transmit_bitrate_kbps_(0), 151 target_delay_ms_(0), 152 network_is_transmitting_(true), 153 encoder_paused_(false), 154 encoder_paused_and_dropped_frame_(false), 155 fec_enabled_(false), 156 nack_enabled_(false), 157 codec_observer_(NULL), 158 effect_filter_(NULL), 159 module_process_thread_(module_process_thread), 160 has_received_sli_(false), 161 picture_id_sli_(0), 162 has_received_rpsi_(false), 163 picture_id_rpsi_(0), 164 qm_callback_(NULL), 165 video_suspended_(false), 166 pre_encode_callback_(NULL) { 167 RtpRtcp::Configuration configuration; 168 configuration.id = ViEModuleId(engine_id_, channel_id_); 169 configuration.audio = false; // Video. 170 171 default_rtp_rtcp_.reset(RtpRtcp::CreateRtpRtcp(configuration)); 172 bitrate_observer_.reset(new ViEBitrateObserver(this)); 173 pacing_callback_.reset(new ViEPacedSenderCallback(this)); 174 paced_sender_.reset( 175 new PacedSender(pacing_callback_.get(), kInitialPace, kPaceMultiplier)); 176 } 177 178 bool ViEEncoder::Init() { 179 if (vcm_.InitializeSender() != 0) { 180 return false; 181 } 182 vpm_.EnableTemporalDecimation(true); 183 184 // Enable/disable content analysis: off by default for now. 185 vpm_.EnableContentAnalysis(false); 186 187 if (module_process_thread_.RegisterModule(&vcm_) != 0 || 188 module_process_thread_.RegisterModule(default_rtp_rtcp_.get()) != 0 || 189 module_process_thread_.RegisterModule(paced_sender_.get()) != 0) { 190 return false; 191 } 192 if (qm_callback_) { 193 delete qm_callback_; 194 } 195 qm_callback_ = new QMVideoSettingsCallback(&vpm_); 196 197 #ifdef VIDEOCODEC_VP8 198 VideoCodec video_codec; 199 if (vcm_.Codec(webrtc::kVideoCodecVP8, &video_codec) != VCM_OK) { 200 return false; 201 } 202 { 203 CriticalSectionScoped cs(data_cs_.get()); 204 send_padding_ = video_codec.numberOfSimulcastStreams > 1; 205 } 206 if (vcm_.RegisterSendCodec(&video_codec, number_of_cores_, 207 default_rtp_rtcp_->MaxDataPayloadLength()) != 0) { 208 return false; 209 } 210 if (default_rtp_rtcp_->RegisterSendPayload(video_codec) != 0) { 211 return false; 212 } 213 #else 214 VideoCodec video_codec; 215 if (vcm_.Codec(webrtc::kVideoCodecI420, &video_codec) == VCM_OK) { 216 { 217 CriticalSectionScoped cs(data_cs_.get()); 218 send_padding_ = video_codec.numberOfSimulcastStreams > 1; 219 } 220 vcm_.RegisterSendCodec(&video_codec, number_of_cores_, 221 default_rtp_rtcp_->MaxDataPayloadLength()); 222 default_rtp_rtcp_->RegisterSendPayload(video_codec); 223 } else { 224 return false; 225 } 226 #endif 227 228 if (vcm_.RegisterTransportCallback(this) != 0) { 229 return false; 230 } 231 if (vcm_.RegisterSendStatisticsCallback(this) != 0) { 232 return false; 233 } 234 if (vcm_.RegisterVideoQMCallback(qm_callback_) != 0) { 235 return false; 236 } 237 return true; 238 } 239 240 ViEEncoder::~ViEEncoder() { 241 if (bitrate_controller_) { 242 bitrate_controller_->RemoveBitrateObserver(bitrate_observer_.get()); 243 } 244 module_process_thread_.DeRegisterModule(&vcm_); 245 module_process_thread_.DeRegisterModule(&vpm_); 246 module_process_thread_.DeRegisterModule(default_rtp_rtcp_.get()); 247 module_process_thread_.DeRegisterModule(paced_sender_.get()); 248 VideoCodingModule::Destroy(&vcm_); 249 VideoProcessingModule::Destroy(&vpm_); 250 delete qm_callback_; 251 } 252 253 int ViEEncoder::Owner() const { 254 return channel_id_; 255 } 256 257 void ViEEncoder::SetNetworkTransmissionState(bool is_transmitting) { 258 { 259 CriticalSectionScoped cs(data_cs_.get()); 260 network_is_transmitting_ = is_transmitting; 261 } 262 if (is_transmitting) { 263 paced_sender_->Resume(); 264 } else { 265 paced_sender_->Pause(); 266 } 267 } 268 269 void ViEEncoder::Pause() { 270 CriticalSectionScoped cs(data_cs_.get()); 271 encoder_paused_ = true; 272 } 273 274 void ViEEncoder::Restart() { 275 CriticalSectionScoped cs(data_cs_.get()); 276 encoder_paused_ = false; 277 } 278 279 uint8_t ViEEncoder::NumberOfCodecs() { 280 return vcm_.NumberOfCodecs(); 281 } 282 283 int32_t ViEEncoder::GetCodec(uint8_t list_index, VideoCodec* video_codec) { 284 if (vcm_.Codec(list_index, video_codec) != 0) { 285 return -1; 286 } 287 return 0; 288 } 289 290 int32_t ViEEncoder::RegisterExternalEncoder(webrtc::VideoEncoder* encoder, 291 uint8_t pl_type, 292 bool internal_source) { 293 if (encoder == NULL) 294 return -1; 295 296 if (vcm_.RegisterExternalEncoder(encoder, pl_type, internal_source) != 297 VCM_OK) { 298 return -1; 299 } 300 return 0; 301 } 302 303 int32_t ViEEncoder::DeRegisterExternalEncoder(uint8_t pl_type) { 304 webrtc::VideoCodec current_send_codec; 305 if (vcm_.SendCodec(¤t_send_codec) == VCM_OK) { 306 uint32_t current_bitrate_bps = 0; 307 if (vcm_.Bitrate(¤t_bitrate_bps) != 0) { 308 LOG(LS_WARNING) << "Failed to get the current encoder target bitrate."; 309 } 310 current_send_codec.startBitrate = (current_bitrate_bps + 500) / 1000; 311 } 312 313 if (vcm_.RegisterExternalEncoder(NULL, pl_type) != VCM_OK) { 314 return -1; 315 } 316 317 // If the external encoder is the current send codec, use vcm internal 318 // encoder. 319 if (current_send_codec.plType == pl_type) { 320 uint16_t max_data_payload_length = 321 default_rtp_rtcp_->MaxDataPayloadLength(); 322 { 323 CriticalSectionScoped cs(data_cs_.get()); 324 send_padding_ = current_send_codec.numberOfSimulcastStreams > 1; 325 } 326 // TODO(mflodman): Unfortunately the VideoCodec that VCM has cached a 327 // raw pointer to an |extra_options| that's long gone. Clearing it here is 328 // a hack to prevent the following code from crashing. This should be fixed 329 // for realz. https://code.google.com/p/chromium/issues/detail?id=348222 330 current_send_codec.extra_options = NULL; 331 if (vcm_.RegisterSendCodec(¤t_send_codec, number_of_cores_, 332 max_data_payload_length) != VCM_OK) { 333 return -1; 334 } 335 } 336 return 0; 337 } 338 339 int32_t ViEEncoder::SetEncoder(const webrtc::VideoCodec& video_codec) { 340 // Setting target width and height for VPM. 341 if (vpm_.SetTargetResolution(video_codec.width, video_codec.height, 342 video_codec.maxFramerate) != VPM_OK) { 343 return -1; 344 } 345 346 if (default_rtp_rtcp_->RegisterSendPayload(video_codec) != 0) { 347 return -1; 348 } 349 // Convert from kbps to bps. 350 std::vector<uint32_t> stream_bitrates = AllocateStreamBitrates( 351 video_codec.startBitrate * 1000, 352 video_codec.simulcastStream, 353 video_codec.numberOfSimulcastStreams); 354 default_rtp_rtcp_->SetTargetSendBitrate(stream_bitrates); 355 356 uint16_t max_data_payload_length = 357 default_rtp_rtcp_->MaxDataPayloadLength(); 358 359 { 360 CriticalSectionScoped cs(data_cs_.get()); 361 send_padding_ = video_codec.numberOfSimulcastStreams > 1; 362 } 363 if (vcm_.RegisterSendCodec(&video_codec, number_of_cores_, 364 max_data_payload_length) != VCM_OK) { 365 return -1; 366 } 367 368 // Set this module as sending right away, let the slave module in the channel 369 // start and stop sending. 370 if (default_rtp_rtcp_->Sending() == false) { 371 if (default_rtp_rtcp_->SetSendingStatus(true) != 0) { 372 return -1; 373 } 374 } 375 bitrate_controller_->SetBitrateObserver(bitrate_observer_.get(), 376 video_codec.startBitrate * 1000, 377 video_codec.minBitrate * 1000, 378 kTransmissionMaxBitrateMultiplier * 379 video_codec.maxBitrate * 1000); 380 381 CriticalSectionScoped crit(data_cs_.get()); 382 int pad_up_to_bitrate_kbps = video_codec.startBitrate; 383 if (pad_up_to_bitrate_kbps < min_transmit_bitrate_kbps_) 384 pad_up_to_bitrate_kbps = min_transmit_bitrate_kbps_; 385 386 paced_sender_->UpdateBitrate(kPaceMultiplier * video_codec.startBitrate, 387 pad_up_to_bitrate_kbps); 388 389 return 0; 390 } 391 392 int32_t ViEEncoder::GetEncoder(VideoCodec* video_codec) { 393 if (vcm_.SendCodec(video_codec) != 0) { 394 return -1; 395 } 396 return 0; 397 } 398 399 int32_t ViEEncoder::GetCodecConfigParameters( 400 unsigned char config_parameters[kConfigParameterSize], 401 unsigned char& config_parameters_size) { 402 int32_t num_parameters = 403 vcm_.CodecConfigParameters(config_parameters, kConfigParameterSize); 404 if (num_parameters <= 0) { 405 config_parameters_size = 0; 406 return -1; 407 } 408 config_parameters_size = static_cast<unsigned char>(num_parameters); 409 return 0; 410 } 411 412 int32_t ViEEncoder::ScaleInputImage(bool enable) { 413 VideoFrameResampling resampling_mode = kFastRescaling; 414 // TODO(mflodman) What? 415 if (enable) { 416 // kInterpolation is currently not supported. 417 LOG_F(LS_ERROR) << "Not supported."; 418 return -1; 419 } 420 vpm_.SetInputFrameResampleMode(resampling_mode); 421 422 return 0; 423 } 424 425 bool ViEEncoder::TimeToSendPacket(uint32_t ssrc, 426 uint16_t sequence_number, 427 int64_t capture_time_ms, 428 bool retransmission) { 429 return default_rtp_rtcp_->TimeToSendPacket(ssrc, sequence_number, 430 capture_time_ms, retransmission); 431 } 432 433 int ViEEncoder::TimeToSendPadding(int bytes) { 434 bool send_padding; 435 { 436 CriticalSectionScoped cs(data_cs_.get()); 437 send_padding = 438 send_padding_ || video_suspended_ || min_transmit_bitrate_kbps_ > 0; 439 } 440 if (send_padding) { 441 return default_rtp_rtcp_->TimeToSendPadding(bytes); 442 } 443 return 0; 444 } 445 446 bool ViEEncoder::EncoderPaused() const { 447 // Pause video if paused by caller or as long as the network is down or the 448 // pacer queue has grown too large in buffered mode. 449 if (encoder_paused_) { 450 return true; 451 } 452 if (target_delay_ms_ > 0) { 453 // Buffered mode. 454 // TODO(pwestin): Workaround until nack is configured as a time and not 455 // number of packets. 456 return paced_sender_->QueueInMs() >= 457 std::max(static_cast<int>(target_delay_ms_ * kEncoderPausePacerMargin), 458 kMinPacingDelayMs); 459 } 460 return !network_is_transmitting_; 461 } 462 463 RtpRtcp* ViEEncoder::SendRtpRtcpModule() { 464 return default_rtp_rtcp_.get(); 465 } 466 467 void ViEEncoder::DeliverFrame(int id, 468 I420VideoFrame* video_frame, 469 int num_csrcs, 470 const uint32_t CSRC[kRtpCsrcSize]) { 471 if (default_rtp_rtcp_->SendingMedia() == false) { 472 // We've paused or we have no channels attached, don't encode. 473 return; 474 } 475 { 476 CriticalSectionScoped cs(data_cs_.get()); 477 time_of_last_incoming_frame_ms_ = TickTime::MillisecondTimestamp(); 478 if (EncoderPaused()) { 479 if (!encoder_paused_and_dropped_frame_) { 480 TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this); 481 } 482 encoder_paused_and_dropped_frame_ = true; 483 return; 484 } 485 if (encoder_paused_and_dropped_frame_) { 486 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); 487 } 488 encoder_paused_and_dropped_frame_ = false; 489 } 490 if (video_frame->native_handle() != NULL) { 491 // TODO(wuchengli): add texture support. http://crbug.com/362437 492 return; 493 } 494 495 // Convert render time, in ms, to RTP timestamp. 496 const int kMsToRtpTimestamp = 90; 497 const uint32_t time_stamp = 498 kMsToRtpTimestamp * 499 static_cast<uint32_t>(video_frame->render_time_ms()); 500 501 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame->render_time_ms(), 502 "Encode"); 503 video_frame->set_timestamp(time_stamp); 504 { 505 CriticalSectionScoped cs(callback_cs_.get()); 506 if (effect_filter_) { 507 unsigned int length = CalcBufferSize(kI420, 508 video_frame->width(), 509 video_frame->height()); 510 scoped_ptr<uint8_t[]> video_buffer(new uint8_t[length]); 511 ExtractBuffer(*video_frame, length, video_buffer.get()); 512 effect_filter_->Transform(length, 513 video_buffer.get(), 514 video_frame->ntp_time_ms(), 515 video_frame->timestamp(), 516 video_frame->width(), 517 video_frame->height()); 518 } 519 } 520 521 // Make sure the CSRC list is correct. 522 if (num_csrcs > 0) { 523 uint32_t tempCSRC[kRtpCsrcSize]; 524 for (int i = 0; i < num_csrcs; i++) { 525 if (CSRC[i] == 1) { 526 tempCSRC[i] = default_rtp_rtcp_->SSRC(); 527 } else { 528 tempCSRC[i] = CSRC[i]; 529 } 530 } 531 default_rtp_rtcp_->SetCSRCs(tempCSRC, (uint8_t) num_csrcs); 532 } 533 // Pass frame via preprocessor. 534 I420VideoFrame* decimated_frame = NULL; 535 const int ret = vpm_.PreprocessFrame(*video_frame, &decimated_frame); 536 if (ret == 1) { 537 // Drop this frame. 538 return; 539 } 540 if (ret != VPM_OK) { 541 return; 542 } 543 // Frame was not sampled => use original. 544 if (decimated_frame == NULL) { 545 decimated_frame = video_frame; 546 } 547 548 { 549 CriticalSectionScoped cs(callback_cs_.get()); 550 if (pre_encode_callback_) 551 pre_encode_callback_->FrameCallback(decimated_frame); 552 } 553 554 #ifdef VIDEOCODEC_VP8 555 if (vcm_.SendCodec() == webrtc::kVideoCodecVP8) { 556 webrtc::CodecSpecificInfo codec_specific_info; 557 codec_specific_info.codecType = webrtc::kVideoCodecVP8; 558 codec_specific_info.codecSpecific.VP8.hasReceivedRPSI = 559 has_received_rpsi_; 560 codec_specific_info.codecSpecific.VP8.hasReceivedSLI = 561 has_received_sli_; 562 codec_specific_info.codecSpecific.VP8.pictureIdRPSI = 563 picture_id_rpsi_; 564 codec_specific_info.codecSpecific.VP8.pictureIdSLI = 565 picture_id_sli_; 566 has_received_sli_ = false; 567 has_received_rpsi_ = false; 568 569 vcm_.AddVideoFrame(*decimated_frame, vpm_.ContentMetrics(), 570 &codec_specific_info); 571 return; 572 } 573 #endif 574 vcm_.AddVideoFrame(*decimated_frame); 575 } 576 577 void ViEEncoder::DelayChanged(int id, int frame_delay) { 578 default_rtp_rtcp_->SetCameraDelay(frame_delay); 579 } 580 581 int ViEEncoder::GetPreferedFrameSettings(int* width, 582 int* height, 583 int* frame_rate) { 584 webrtc::VideoCodec video_codec; 585 memset(&video_codec, 0, sizeof(video_codec)); 586 if (vcm_.SendCodec(&video_codec) != VCM_OK) { 587 return -1; 588 } 589 590 *width = video_codec.width; 591 *height = video_codec.height; 592 *frame_rate = video_codec.maxFramerate; 593 return 0; 594 } 595 596 int ViEEncoder::SendKeyFrame() { 597 return vcm_.IntraFrameRequest(0); 598 } 599 600 int32_t ViEEncoder::SendCodecStatistics( 601 uint32_t* num_key_frames, uint32_t* num_delta_frames) { 602 webrtc::VCMFrameCount sent_frames; 603 if (vcm_.SentFrameCount(sent_frames) != VCM_OK) { 604 return -1; 605 } 606 *num_key_frames = sent_frames.numKeyFrames; 607 *num_delta_frames = sent_frames.numDeltaFrames; 608 return 0; 609 } 610 611 int32_t ViEEncoder::PacerQueuingDelayMs() const { 612 return paced_sender_->QueueInMs(); 613 } 614 615 int ViEEncoder::CodecTargetBitrate(uint32_t* bitrate) const { 616 if (vcm_.Bitrate(bitrate) != 0) 617 return -1; 618 return 0; 619 } 620 621 int32_t ViEEncoder::UpdateProtectionMethod(bool enable_nack) { 622 bool fec_enabled = false; 623 uint8_t dummy_ptype_red = 0; 624 uint8_t dummy_ptypeFEC = 0; 625 626 // Updated protection method to VCM to get correct packetization sizes. 627 // FEC has larger overhead than NACK -> set FEC if used. 628 int32_t error = default_rtp_rtcp_->GenericFECStatus(fec_enabled, 629 dummy_ptype_red, 630 dummy_ptypeFEC); 631 if (error) { 632 return -1; 633 } 634 if (fec_enabled_ == fec_enabled && nack_enabled_ == enable_nack) { 635 // No change needed, we're already in correct state. 636 return 0; 637 } 638 fec_enabled_ = fec_enabled; 639 nack_enabled_ = enable_nack; 640 641 // Set Video Protection for VCM. 642 if (fec_enabled && nack_enabled_) { 643 vcm_.SetVideoProtection(webrtc::kProtectionNackFEC, true); 644 } else { 645 vcm_.SetVideoProtection(webrtc::kProtectionFEC, fec_enabled_); 646 vcm_.SetVideoProtection(webrtc::kProtectionNackSender, nack_enabled_); 647 vcm_.SetVideoProtection(webrtc::kProtectionNackFEC, false); 648 } 649 650 if (fec_enabled_ || nack_enabled_) { 651 vcm_.RegisterProtectionCallback(this); 652 // The send codec must be registered to set correct MTU. 653 webrtc::VideoCodec codec; 654 if (vcm_.SendCodec(&codec) == 0) { 655 uint16_t max_pay_load = default_rtp_rtcp_->MaxDataPayloadLength(); 656 uint32_t current_bitrate_bps = 0; 657 if (vcm_.Bitrate(¤t_bitrate_bps) != 0) { 658 LOG_F(LS_WARNING) << 659 "Failed to get the current encoder target bitrate."; 660 } 661 // Convert to start bitrate in kbps. 662 codec.startBitrate = (current_bitrate_bps + 500) / 1000; 663 if (vcm_.RegisterSendCodec(&codec, number_of_cores_, max_pay_load) != 0) { 664 return -1; 665 } 666 } 667 return 0; 668 } else { 669 // FEC and NACK are disabled. 670 vcm_.RegisterProtectionCallback(NULL); 671 } 672 return 0; 673 } 674 675 void ViEEncoder::SetSenderBufferingMode(int target_delay_ms) { 676 { 677 CriticalSectionScoped cs(data_cs_.get()); 678 target_delay_ms_ = target_delay_ms; 679 } 680 if (target_delay_ms > 0) { 681 // Disable external frame-droppers. 682 vcm_.EnableFrameDropper(false); 683 vpm_.EnableTemporalDecimation(false); 684 // We don't put any limits on the pacer queue when running in buffered mode 685 // since the encoder will be paused if the queue grow too large. 686 paced_sender_->set_max_queue_length_ms(-1); 687 } else { 688 // Real-time mode - enable frame droppers. 689 vpm_.EnableTemporalDecimation(true); 690 vcm_.EnableFrameDropper(true); 691 paced_sender_->set_max_queue_length_ms( 692 PacedSender::kDefaultMaxQueueLengthMs); 693 } 694 } 695 696 int32_t ViEEncoder::SendData( 697 const FrameType frame_type, 698 const uint8_t payload_type, 699 const uint32_t time_stamp, 700 int64_t capture_time_ms, 701 const uint8_t* payload_data, 702 const uint32_t payload_size, 703 const webrtc::RTPFragmentationHeader& fragmentation_header, 704 const RTPVideoHeader* rtp_video_hdr) { 705 // New encoded data, hand over to the rtp module. 706 return default_rtp_rtcp_->SendOutgoingData(frame_type, 707 payload_type, 708 time_stamp, 709 capture_time_ms, 710 payload_data, 711 payload_size, 712 &fragmentation_header, 713 rtp_video_hdr); 714 } 715 716 int32_t ViEEncoder::ProtectionRequest( 717 const FecProtectionParams* delta_fec_params, 718 const FecProtectionParams* key_fec_params, 719 uint32_t* sent_video_rate_bps, 720 uint32_t* sent_nack_rate_bps, 721 uint32_t* sent_fec_rate_bps) { 722 default_rtp_rtcp_->SetFecParameters(delta_fec_params, key_fec_params); 723 default_rtp_rtcp_->BitrateSent(NULL, sent_video_rate_bps, sent_fec_rate_bps, 724 sent_nack_rate_bps); 725 return 0; 726 } 727 728 int32_t ViEEncoder::SendStatistics(const uint32_t bit_rate, 729 const uint32_t frame_rate) { 730 CriticalSectionScoped cs(callback_cs_.get()); 731 if (codec_observer_) { 732 codec_observer_->OutgoingRate(channel_id_, frame_rate, bit_rate); 733 } 734 return 0; 735 } 736 737 int32_t ViEEncoder::RegisterCodecObserver(ViEEncoderObserver* observer) { 738 CriticalSectionScoped cs(callback_cs_.get()); 739 if (observer && codec_observer_) { 740 LOG_F(LS_ERROR) << "Observer already set."; 741 return -1; 742 } 743 codec_observer_ = observer; 744 return 0; 745 } 746 747 void ViEEncoder::OnReceivedSLI(uint32_t /*ssrc*/, 748 uint8_t picture_id) { 749 picture_id_sli_ = picture_id; 750 has_received_sli_ = true; 751 } 752 753 void ViEEncoder::OnReceivedRPSI(uint32_t /*ssrc*/, 754 uint64_t picture_id) { 755 picture_id_rpsi_ = picture_id; 756 has_received_rpsi_ = true; 757 } 758 759 void ViEEncoder::OnReceivedIntraFrameRequest(uint32_t ssrc) { 760 // Key frame request from remote side, signal to VCM. 761 TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); 762 763 int idx = 0; 764 { 765 CriticalSectionScoped cs(data_cs_.get()); 766 std::map<unsigned int, int>::iterator stream_it = ssrc_streams_.find(ssrc); 767 if (stream_it == ssrc_streams_.end()) { 768 LOG_F(LS_WARNING) << "ssrc not found: " << ssrc << ", map size " 769 << ssrc_streams_.size(); 770 return; 771 } 772 std::map<unsigned int, int64_t>::iterator time_it = 773 time_last_intra_request_ms_.find(ssrc); 774 if (time_it == time_last_intra_request_ms_.end()) { 775 time_last_intra_request_ms_[ssrc] = 0; 776 } 777 778 int64_t now = TickTime::MillisecondTimestamp(); 779 if (time_last_intra_request_ms_[ssrc] + kViEMinKeyRequestIntervalMs > now) { 780 return; 781 } 782 time_last_intra_request_ms_[ssrc] = now; 783 idx = stream_it->second; 784 } 785 // Release the critsect before triggering key frame. 786 vcm_.IntraFrameRequest(idx); 787 } 788 789 void ViEEncoder::OnLocalSsrcChanged(uint32_t old_ssrc, uint32_t new_ssrc) { 790 CriticalSectionScoped cs(data_cs_.get()); 791 std::map<unsigned int, int>::iterator it = ssrc_streams_.find(old_ssrc); 792 if (it == ssrc_streams_.end()) { 793 return; 794 } 795 796 ssrc_streams_[new_ssrc] = it->second; 797 ssrc_streams_.erase(it); 798 799 std::map<unsigned int, int64_t>::iterator time_it = 800 time_last_intra_request_ms_.find(old_ssrc); 801 int64_t last_intra_request_ms = 0; 802 if (time_it != time_last_intra_request_ms_.end()) { 803 last_intra_request_ms = time_it->second; 804 time_last_intra_request_ms_.erase(time_it); 805 } 806 time_last_intra_request_ms_[new_ssrc] = last_intra_request_ms; 807 } 808 809 bool ViEEncoder::SetSsrcs(const std::list<unsigned int>& ssrcs) { 810 VideoCodec codec; 811 if (vcm_.SendCodec(&codec) != 0) 812 return false; 813 814 if (codec.numberOfSimulcastStreams > 0 && 815 ssrcs.size() != codec.numberOfSimulcastStreams) { 816 return false; 817 } 818 819 CriticalSectionScoped cs(data_cs_.get()); 820 ssrc_streams_.clear(); 821 time_last_intra_request_ms_.clear(); 822 int idx = 0; 823 for (std::list<unsigned int>::const_iterator it = ssrcs.begin(); 824 it != ssrcs.end(); ++it, ++idx) { 825 unsigned int ssrc = *it; 826 ssrc_streams_[ssrc] = idx; 827 } 828 return true; 829 } 830 831 void ViEEncoder::SetMinTransmitBitrate(int min_transmit_bitrate_kbps) { 832 assert(min_transmit_bitrate_kbps >= 0); 833 CriticalSectionScoped crit(data_cs_.get()); 834 min_transmit_bitrate_kbps_ = min_transmit_bitrate_kbps; 835 } 836 837 // Called from ViEBitrateObserver. 838 void ViEEncoder::OnNetworkChanged(const uint32_t bitrate_bps, 839 const uint8_t fraction_lost, 840 const uint32_t round_trip_time_ms) { 841 LOG(LS_VERBOSE) << "OnNetworkChanged, bitrate" << bitrate_bps 842 << " packet loss " << fraction_lost 843 << " rtt " << round_trip_time_ms; 844 vcm_.SetChannelParameters(bitrate_bps, fraction_lost, round_trip_time_ms); 845 bool video_is_suspended = vcm_.VideoSuspended(); 846 int bitrate_kbps = bitrate_bps / 1000; 847 VideoCodec send_codec; 848 if (vcm_.SendCodec(&send_codec) != 0) { 849 return; 850 } 851 SimulcastStream* stream_configs = send_codec.simulcastStream; 852 // Allocate the bandwidth between the streams. 853 std::vector<uint32_t> stream_bitrates = AllocateStreamBitrates( 854 bitrate_bps, 855 stream_configs, 856 send_codec.numberOfSimulcastStreams); 857 // Find the max amount of padding we can allow ourselves to send at this 858 // point, based on which streams are currently active and what our current 859 // available bandwidth is. 860 int pad_up_to_bitrate_kbps = 0; 861 if (send_codec.numberOfSimulcastStreams == 0) { 862 pad_up_to_bitrate_kbps = send_codec.minBitrate; 863 } else { 864 pad_up_to_bitrate_kbps = 865 stream_configs[send_codec.numberOfSimulcastStreams - 1].minBitrate; 866 for (int i = 0; i < send_codec.numberOfSimulcastStreams - 1; ++i) { 867 pad_up_to_bitrate_kbps += stream_configs[i].targetBitrate; 868 } 869 } 870 871 // Disable padding if only sending one stream and video isn't suspended and 872 // min-transmit bitrate isn't used (applied later). 873 if (!video_is_suspended && send_codec.numberOfSimulcastStreams <= 1) 874 pad_up_to_bitrate_kbps = 0; 875 876 { 877 CriticalSectionScoped cs(data_cs_.get()); 878 // The amount of padding should decay to zero if no frames are being 879 // captured unless a min-transmit bitrate is used. 880 int64_t now_ms = TickTime::MillisecondTimestamp(); 881 if (now_ms - time_of_last_incoming_frame_ms_ > kStopPaddingThresholdMs) 882 pad_up_to_bitrate_kbps = 0; 883 884 // Pad up to min bitrate. 885 if (pad_up_to_bitrate_kbps < min_transmit_bitrate_kbps_) 886 pad_up_to_bitrate_kbps = min_transmit_bitrate_kbps_; 887 888 // Padding may never exceed bitrate estimate. 889 if (pad_up_to_bitrate_kbps > bitrate_kbps) 890 pad_up_to_bitrate_kbps = bitrate_kbps; 891 892 paced_sender_->UpdateBitrate(kPaceMultiplier * bitrate_kbps, 893 pad_up_to_bitrate_kbps); 894 default_rtp_rtcp_->SetTargetSendBitrate(stream_bitrates); 895 if (video_suspended_ == video_is_suspended) 896 return; 897 video_suspended_ = video_is_suspended; 898 } 899 900 // Video suspend-state changed, inform codec observer. 901 CriticalSectionScoped crit(callback_cs_.get()); 902 if (codec_observer_) { 903 LOG(LS_INFO) << "Video suspended " << video_is_suspended 904 << " for channel " << channel_id_; 905 codec_observer_->SuspendChange(channel_id_, video_is_suspended); 906 } 907 } 908 909 PacedSender* ViEEncoder::GetPacedSender() { 910 return paced_sender_.get(); 911 } 912 913 int32_t ViEEncoder::RegisterEffectFilter(ViEEffectFilter* effect_filter) { 914 CriticalSectionScoped cs(callback_cs_.get()); 915 if (effect_filter != NULL && effect_filter_ != NULL) { 916 LOG_F(LS_ERROR) << "Filter already set."; 917 return -1; 918 } 919 effect_filter_ = effect_filter; 920 return 0; 921 } 922 923 int ViEEncoder::StartDebugRecording(const char* fileNameUTF8) { 924 return vcm_.StartDebugRecording(fileNameUTF8); 925 } 926 927 int ViEEncoder::StopDebugRecording() { 928 return vcm_.StopDebugRecording(); 929 } 930 931 void ViEEncoder::SuspendBelowMinBitrate() { 932 vcm_.SuspendBelowMinBitrate(); 933 bitrate_controller_->EnforceMinBitrate(false); 934 } 935 936 void ViEEncoder::RegisterPreEncodeCallback( 937 I420FrameCallback* pre_encode_callback) { 938 CriticalSectionScoped cs(callback_cs_.get()); 939 pre_encode_callback_ = pre_encode_callback; 940 } 941 942 void ViEEncoder::DeRegisterPreEncodeCallback() { 943 CriticalSectionScoped cs(callback_cs_.get()); 944 pre_encode_callback_ = NULL; 945 } 946 947 void ViEEncoder::RegisterPostEncodeImageCallback( 948 EncodedImageCallback* post_encode_callback) { 949 vcm_.RegisterPostEncodeImageCallback(post_encode_callback); 950 } 951 952 void ViEEncoder::DeRegisterPostEncodeImageCallback() { 953 vcm_.RegisterPostEncodeImageCallback(NULL); 954 } 955 956 QMVideoSettingsCallback::QMVideoSettingsCallback(VideoProcessingModule* vpm) 957 : vpm_(vpm) { 958 } 959 960 QMVideoSettingsCallback::~QMVideoSettingsCallback() { 961 } 962 963 int32_t QMVideoSettingsCallback::SetVideoQMSettings( 964 const uint32_t frame_rate, 965 const uint32_t width, 966 const uint32_t height) { 967 return vpm_->SetTargetResolution(width, height, frame_rate); 968 } 969 970 } // namespace webrtc 971