1 /* 2 * libjingle 3 * Copyright 2014 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #ifdef HAVE_WEBRTC_VIDEO 29 #include "talk/media/webrtc/webrtcvideoengine2.h" 30 31 #include <algorithm> 32 #include <set> 33 #include <string> 34 35 #include "talk/media/base/videocapturer.h" 36 #include "talk/media/base/videorenderer.h" 37 #include "talk/media/webrtc/constants.h" 38 #include "talk/media/webrtc/simulcast.h" 39 #include "talk/media/webrtc/webrtcmediaengine.h" 40 #include "talk/media/webrtc/webrtcvideoencoderfactory.h" 41 #include "talk/media/webrtc/webrtcvideoframe.h" 42 #include "talk/media/webrtc/webrtcvoiceengine.h" 43 #include "webrtc/base/buffer.h" 44 #include "webrtc/base/logging.h" 45 #include "webrtc/base/stringutils.h" 46 #include "webrtc/base/timeutils.h" 47 #include "webrtc/base/trace_event.h" 48 #include "webrtc/call.h" 49 #include "webrtc/modules/video_coding/codecs/h264/include/h264.h" 50 #include "webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h" 51 #include "webrtc/system_wrappers/include/field_trial.h" 52 #include "webrtc/video_decoder.h" 53 #include "webrtc/video_encoder.h" 54 55 namespace cricket { 56 namespace { 57 58 // Wrap cricket::WebRtcVideoEncoderFactory as a webrtc::VideoEncoderFactory. 59 class EncoderFactoryAdapter : public webrtc::VideoEncoderFactory { 60 public: 61 // EncoderFactoryAdapter doesn't take ownership of |factory|, which is owned 62 // by e.g. PeerConnectionFactory. 63 explicit EncoderFactoryAdapter(cricket::WebRtcVideoEncoderFactory* factory) 64 : factory_(factory) {} 65 virtual ~EncoderFactoryAdapter() {} 66 67 // Implement webrtc::VideoEncoderFactory. 68 webrtc::VideoEncoder* Create() override { 69 return factory_->CreateVideoEncoder(webrtc::kVideoCodecVP8); 70 } 71 72 void Destroy(webrtc::VideoEncoder* encoder) override { 73 return factory_->DestroyVideoEncoder(encoder); 74 } 75 76 private: 77 cricket::WebRtcVideoEncoderFactory* const factory_; 78 }; 79 80 // An encoder factory that wraps Create requests for simulcastable codec types 81 // with a webrtc::SimulcastEncoderAdapter. Non simulcastable codec type 82 // requests are just passed through to the contained encoder factory. 83 class WebRtcSimulcastEncoderFactory 84 : public cricket::WebRtcVideoEncoderFactory { 85 public: 86 // WebRtcSimulcastEncoderFactory doesn't take ownership of |factory|, which is 87 // owned by e.g. PeerConnectionFactory. 88 explicit WebRtcSimulcastEncoderFactory( 89 cricket::WebRtcVideoEncoderFactory* factory) 90 : factory_(factory) {} 91 92 static bool UseSimulcastEncoderFactory( 93 const std::vector<VideoCodec>& codecs) { 94 // If any codec is VP8, use the simulcast factory. If asked to create a 95 // non-VP8 codec, we'll just return a contained factory encoder directly. 96 for (const auto& codec : codecs) { 97 if (codec.type == webrtc::kVideoCodecVP8) { 98 return true; 99 } 100 } 101 return false; 102 } 103 104 webrtc::VideoEncoder* CreateVideoEncoder( 105 webrtc::VideoCodecType type) override { 106 RTC_DCHECK(factory_ != NULL); 107 // If it's a codec type we can simulcast, create a wrapped encoder. 108 if (type == webrtc::kVideoCodecVP8) { 109 return new webrtc::SimulcastEncoderAdapter( 110 new EncoderFactoryAdapter(factory_)); 111 } 112 webrtc::VideoEncoder* encoder = factory_->CreateVideoEncoder(type); 113 if (encoder) { 114 non_simulcast_encoders_.push_back(encoder); 115 } 116 return encoder; 117 } 118 119 const std::vector<VideoCodec>& codecs() const override { 120 return factory_->codecs(); 121 } 122 123 bool EncoderTypeHasInternalSource( 124 webrtc::VideoCodecType type) const override { 125 return factory_->EncoderTypeHasInternalSource(type); 126 } 127 128 void DestroyVideoEncoder(webrtc::VideoEncoder* encoder) override { 129 // Check first to see if the encoder wasn't wrapped in a 130 // SimulcastEncoderAdapter. In that case, ask the factory to destroy it. 131 if (std::remove(non_simulcast_encoders_.begin(), 132 non_simulcast_encoders_.end(), 133 encoder) != non_simulcast_encoders_.end()) { 134 factory_->DestroyVideoEncoder(encoder); 135 return; 136 } 137 138 // Otherwise, SimulcastEncoderAdapter can be deleted directly, and will call 139 // DestroyVideoEncoder on the factory for individual encoder instances. 140 delete encoder; 141 } 142 143 private: 144 cricket::WebRtcVideoEncoderFactory* factory_; 145 // A list of encoders that were created without being wrapped in a 146 // SimulcastEncoderAdapter. 147 std::vector<webrtc::VideoEncoder*> non_simulcast_encoders_; 148 }; 149 150 bool CodecIsInternallySupported(const std::string& codec_name) { 151 if (CodecNamesEq(codec_name, kVp8CodecName)) { 152 return true; 153 } 154 if (CodecNamesEq(codec_name, kVp9CodecName)) { 155 return true; 156 } 157 if (CodecNamesEq(codec_name, kH264CodecName)) { 158 return webrtc::H264Encoder::IsSupported() && 159 webrtc::H264Decoder::IsSupported(); 160 } 161 return false; 162 } 163 164 void AddDefaultFeedbackParams(VideoCodec* codec) { 165 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamCcm, kRtcpFbCcmParamFir)); 166 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kParamValueEmpty)); 167 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kRtcpFbNackParamPli)); 168 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamRemb, kParamValueEmpty)); 169 codec->AddFeedbackParam( 170 FeedbackParam(kRtcpFbParamTransportCc, kParamValueEmpty)); 171 } 172 173 static VideoCodec MakeVideoCodecWithDefaultFeedbackParams(int payload_type, 174 const char* name) { 175 VideoCodec codec(payload_type, name, kDefaultVideoMaxWidth, 176 kDefaultVideoMaxHeight, kDefaultVideoMaxFramerate, 0); 177 AddDefaultFeedbackParams(&codec); 178 return codec; 179 } 180 181 static std::string CodecVectorToString(const std::vector<VideoCodec>& codecs) { 182 std::stringstream out; 183 out << '{'; 184 for (size_t i = 0; i < codecs.size(); ++i) { 185 out << codecs[i].ToString(); 186 if (i != codecs.size() - 1) { 187 out << ", "; 188 } 189 } 190 out << '}'; 191 return out.str(); 192 } 193 194 static bool ValidateCodecFormats(const std::vector<VideoCodec>& codecs) { 195 bool has_video = false; 196 for (size_t i = 0; i < codecs.size(); ++i) { 197 if (!codecs[i].ValidateCodecFormat()) { 198 return false; 199 } 200 if (codecs[i].GetCodecType() == VideoCodec::CODEC_VIDEO) { 201 has_video = true; 202 } 203 } 204 if (!has_video) { 205 LOG(LS_ERROR) << "Setting codecs without a video codec is invalid: " 206 << CodecVectorToString(codecs); 207 return false; 208 } 209 return true; 210 } 211 212 static bool ValidateStreamParams(const StreamParams& sp) { 213 if (sp.ssrcs.empty()) { 214 LOG(LS_ERROR) << "No SSRCs in stream parameters: " << sp.ToString(); 215 return false; 216 } 217 218 std::vector<uint32_t> primary_ssrcs; 219 sp.GetPrimarySsrcs(&primary_ssrcs); 220 std::vector<uint32_t> rtx_ssrcs; 221 sp.GetFidSsrcs(primary_ssrcs, &rtx_ssrcs); 222 for (uint32_t rtx_ssrc : rtx_ssrcs) { 223 bool rtx_ssrc_present = false; 224 for (uint32_t sp_ssrc : sp.ssrcs) { 225 if (sp_ssrc == rtx_ssrc) { 226 rtx_ssrc_present = true; 227 break; 228 } 229 } 230 if (!rtx_ssrc_present) { 231 LOG(LS_ERROR) << "RTX SSRC '" << rtx_ssrc 232 << "' missing from StreamParams ssrcs: " << sp.ToString(); 233 return false; 234 } 235 } 236 if (!rtx_ssrcs.empty() && primary_ssrcs.size() != rtx_ssrcs.size()) { 237 LOG(LS_ERROR) 238 << "RTX SSRCs exist, but don't cover all SSRCs (unsupported): " 239 << sp.ToString(); 240 return false; 241 } 242 243 return true; 244 } 245 246 inline const webrtc::RtpExtension* FindHeaderExtension( 247 const std::vector<webrtc::RtpExtension>& extensions, 248 const std::string& name) { 249 for (const auto& kv : extensions) { 250 if (kv.name == name) { 251 return &kv; 252 } 253 } 254 return NULL; 255 } 256 257 // Merges two fec configs and logs an error if a conflict arises 258 // such that merging in different order would trigger a different output. 259 static void MergeFecConfig(const webrtc::FecConfig& other, 260 webrtc::FecConfig* output) { 261 if (other.ulpfec_payload_type != -1) { 262 if (output->ulpfec_payload_type != -1 && 263 output->ulpfec_payload_type != other.ulpfec_payload_type) { 264 LOG(LS_WARNING) << "Conflict merging ulpfec_payload_type configs: " 265 << output->ulpfec_payload_type << " and " 266 << other.ulpfec_payload_type; 267 } 268 output->ulpfec_payload_type = other.ulpfec_payload_type; 269 } 270 if (other.red_payload_type != -1) { 271 if (output->red_payload_type != -1 && 272 output->red_payload_type != other.red_payload_type) { 273 LOG(LS_WARNING) << "Conflict merging red_payload_type configs: " 274 << output->red_payload_type << " and " 275 << other.red_payload_type; 276 } 277 output->red_payload_type = other.red_payload_type; 278 } 279 if (other.red_rtx_payload_type != -1) { 280 if (output->red_rtx_payload_type != -1 && 281 output->red_rtx_payload_type != other.red_rtx_payload_type) { 282 LOG(LS_WARNING) << "Conflict merging red_rtx_payload_type configs: " 283 << output->red_rtx_payload_type << " and " 284 << other.red_rtx_payload_type; 285 } 286 output->red_rtx_payload_type = other.red_rtx_payload_type; 287 } 288 } 289 290 // Returns true if the given codec is disallowed from doing simulcast. 291 bool IsCodecBlacklistedForSimulcast(const std::string& codec_name) { 292 return CodecNamesEq(codec_name, kH264CodecName) || 293 CodecNamesEq(codec_name, kVp9CodecName); 294 } 295 296 // The selected thresholds for QVGA and VGA corresponded to a QP around 10. 297 // The change in QP declined above the selected bitrates. 298 static int GetMaxDefaultVideoBitrateKbps(int width, int height) { 299 if (width * height <= 320 * 240) { 300 return 600; 301 } else if (width * height <= 640 * 480) { 302 return 1700; 303 } else if (width * height <= 960 * 540) { 304 return 2000; 305 } else { 306 return 2500; 307 } 308 } 309 } // namespace 310 311 // Constants defined in talk/media/webrtc/constants.h 312 // TODO(pbos): Move these to a separate constants.cc file. 313 const int kMinVideoBitrate = 30; 314 const int kStartVideoBitrate = 300; 315 316 const int kVideoMtu = 1200; 317 const int kVideoRtpBufferSize = 65536; 318 319 // This constant is really an on/off, lower-level configurable NACK history 320 // duration hasn't been implemented. 321 static const int kNackHistoryMs = 1000; 322 323 static const int kDefaultQpMax = 56; 324 325 static const int kDefaultRtcpReceiverReportSsrc = 1; 326 327 std::vector<VideoCodec> DefaultVideoCodecList() { 328 std::vector<VideoCodec> codecs; 329 codecs.push_back(MakeVideoCodecWithDefaultFeedbackParams(kDefaultVp8PlType, 330 kVp8CodecName)); 331 if (CodecIsInternallySupported(kVp9CodecName)) { 332 codecs.push_back(MakeVideoCodecWithDefaultFeedbackParams(kDefaultVp9PlType, 333 kVp9CodecName)); 334 // TODO(andresp): Add rtx codec for vp9 and verify it works. 335 } 336 if (CodecIsInternallySupported(kH264CodecName)) { 337 codecs.push_back(MakeVideoCodecWithDefaultFeedbackParams(kDefaultH264PlType, 338 kH264CodecName)); 339 } 340 codecs.push_back( 341 VideoCodec::CreateRtxCodec(kDefaultRtxVp8PlType, kDefaultVp8PlType)); 342 codecs.push_back(VideoCodec(kDefaultRedPlType, kRedCodecName)); 343 codecs.push_back(VideoCodec(kDefaultUlpfecType, kUlpfecCodecName)); 344 return codecs; 345 } 346 347 std::vector<webrtc::VideoStream> 348 WebRtcVideoChannel2::WebRtcVideoSendStream::CreateSimulcastVideoStreams( 349 const VideoCodec& codec, 350 const VideoOptions& options, 351 int max_bitrate_bps, 352 size_t num_streams) { 353 int max_qp = kDefaultQpMax; 354 codec.GetParam(kCodecParamMaxQuantization, &max_qp); 355 356 return GetSimulcastConfig( 357 num_streams, codec.width, codec.height, max_bitrate_bps, max_qp, 358 codec.framerate != 0 ? codec.framerate : kDefaultVideoMaxFramerate); 359 } 360 361 std::vector<webrtc::VideoStream> 362 WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoStreams( 363 const VideoCodec& codec, 364 const VideoOptions& options, 365 int max_bitrate_bps, 366 size_t num_streams) { 367 int codec_max_bitrate_kbps; 368 if (codec.GetParam(kCodecParamMaxBitrate, &codec_max_bitrate_kbps)) { 369 max_bitrate_bps = codec_max_bitrate_kbps * 1000; 370 } 371 if (num_streams != 1) { 372 return CreateSimulcastVideoStreams(codec, options, max_bitrate_bps, 373 num_streams); 374 } 375 376 // For unset max bitrates set default bitrate for non-simulcast. 377 if (max_bitrate_bps <= 0) { 378 max_bitrate_bps = 379 GetMaxDefaultVideoBitrateKbps(codec.width, codec.height) * 1000; 380 } 381 382 webrtc::VideoStream stream; 383 stream.width = codec.width; 384 stream.height = codec.height; 385 stream.max_framerate = 386 codec.framerate != 0 ? codec.framerate : kDefaultVideoMaxFramerate; 387 388 stream.min_bitrate_bps = kMinVideoBitrate * 1000; 389 stream.target_bitrate_bps = stream.max_bitrate_bps = max_bitrate_bps; 390 391 int max_qp = kDefaultQpMax; 392 codec.GetParam(kCodecParamMaxQuantization, &max_qp); 393 stream.max_qp = max_qp; 394 std::vector<webrtc::VideoStream> streams; 395 streams.push_back(stream); 396 return streams; 397 } 398 399 void* WebRtcVideoChannel2::WebRtcVideoSendStream::ConfigureVideoEncoderSettings( 400 const VideoCodec& codec, 401 const VideoOptions& options, 402 bool is_screencast) { 403 // No automatic resizing when using simulcast or screencast. 404 bool automatic_resize = 405 !is_screencast && parameters_.config.rtp.ssrcs.size() == 1; 406 bool frame_dropping = !is_screencast; 407 bool denoising; 408 bool codec_default_denoising = false; 409 if (is_screencast) { 410 denoising = false; 411 } else { 412 // Use codec default if video_noise_reduction is unset. 413 codec_default_denoising = !options.video_noise_reduction; 414 denoising = options.video_noise_reduction.value_or(false); 415 } 416 417 if (CodecNamesEq(codec.name, kVp8CodecName)) { 418 encoder_settings_.vp8 = webrtc::VideoEncoder::GetDefaultVp8Settings(); 419 encoder_settings_.vp8.automaticResizeOn = automatic_resize; 420 // VP8 denoising is enabled by default. 421 encoder_settings_.vp8.denoisingOn = 422 codec_default_denoising ? true : denoising; 423 encoder_settings_.vp8.frameDroppingOn = frame_dropping; 424 return &encoder_settings_.vp8; 425 } 426 if (CodecNamesEq(codec.name, kVp9CodecName)) { 427 encoder_settings_.vp9 = webrtc::VideoEncoder::GetDefaultVp9Settings(); 428 // VP9 denoising is disabled by default. 429 encoder_settings_.vp9.denoisingOn = 430 codec_default_denoising ? false : denoising; 431 encoder_settings_.vp9.frameDroppingOn = frame_dropping; 432 return &encoder_settings_.vp9; 433 } 434 return NULL; 435 } 436 437 DefaultUnsignalledSsrcHandler::DefaultUnsignalledSsrcHandler() 438 : default_recv_ssrc_(0), default_renderer_(NULL) {} 439 440 UnsignalledSsrcHandler::Action DefaultUnsignalledSsrcHandler::OnUnsignalledSsrc( 441 WebRtcVideoChannel2* channel, 442 uint32_t ssrc) { 443 if (default_recv_ssrc_ != 0) { // Already one default stream. 444 LOG(LS_WARNING) << "Unknown SSRC, but default receive stream already set."; 445 return kDropPacket; 446 } 447 448 StreamParams sp; 449 sp.ssrcs.push_back(ssrc); 450 LOG(LS_INFO) << "Creating default receive stream for SSRC=" << ssrc << "."; 451 if (!channel->AddRecvStream(sp, true)) { 452 LOG(LS_WARNING) << "Could not create default receive stream."; 453 } 454 455 channel->SetRenderer(ssrc, default_renderer_); 456 default_recv_ssrc_ = ssrc; 457 return kDeliverPacket; 458 } 459 460 VideoRenderer* DefaultUnsignalledSsrcHandler::GetDefaultRenderer() const { 461 return default_renderer_; 462 } 463 464 void DefaultUnsignalledSsrcHandler::SetDefaultRenderer( 465 VideoMediaChannel* channel, 466 VideoRenderer* renderer) { 467 default_renderer_ = renderer; 468 if (default_recv_ssrc_ != 0) { 469 channel->SetRenderer(default_recv_ssrc_, default_renderer_); 470 } 471 } 472 473 WebRtcVideoEngine2::WebRtcVideoEngine2() 474 : initialized_(false), 475 external_decoder_factory_(NULL), 476 external_encoder_factory_(NULL) { 477 LOG(LS_INFO) << "WebRtcVideoEngine2::WebRtcVideoEngine2()"; 478 video_codecs_ = GetSupportedCodecs(); 479 } 480 481 WebRtcVideoEngine2::~WebRtcVideoEngine2() { 482 LOG(LS_INFO) << "WebRtcVideoEngine2::~WebRtcVideoEngine2"; 483 } 484 485 void WebRtcVideoEngine2::Init() { 486 LOG(LS_INFO) << "WebRtcVideoEngine2::Init"; 487 initialized_ = true; 488 } 489 490 WebRtcVideoChannel2* WebRtcVideoEngine2::CreateChannel( 491 webrtc::Call* call, 492 const VideoOptions& options) { 493 RTC_DCHECK(initialized_); 494 LOG(LS_INFO) << "CreateChannel. Options: " << options.ToString(); 495 return new WebRtcVideoChannel2(call, options, video_codecs_, 496 external_encoder_factory_, external_decoder_factory_); 497 } 498 499 const std::vector<VideoCodec>& WebRtcVideoEngine2::codecs() const { 500 return video_codecs_; 501 } 502 503 RtpCapabilities WebRtcVideoEngine2::GetCapabilities() const { 504 RtpCapabilities capabilities; 505 capabilities.header_extensions.push_back( 506 RtpHeaderExtension(kRtpTimestampOffsetHeaderExtension, 507 kRtpTimestampOffsetHeaderExtensionDefaultId)); 508 capabilities.header_extensions.push_back( 509 RtpHeaderExtension(kRtpAbsoluteSenderTimeHeaderExtension, 510 kRtpAbsoluteSenderTimeHeaderExtensionDefaultId)); 511 capabilities.header_extensions.push_back( 512 RtpHeaderExtension(kRtpVideoRotationHeaderExtension, 513 kRtpVideoRotationHeaderExtensionDefaultId)); 514 if (webrtc::field_trial::FindFullName("WebRTC-SendSideBwe") == "Enabled") { 515 capabilities.header_extensions.push_back(RtpHeaderExtension( 516 kRtpTransportSequenceNumberHeaderExtension, 517 kRtpTransportSequenceNumberHeaderExtensionDefaultId)); 518 } 519 return capabilities; 520 } 521 522 void WebRtcVideoEngine2::SetExternalDecoderFactory( 523 WebRtcVideoDecoderFactory* decoder_factory) { 524 RTC_DCHECK(!initialized_); 525 external_decoder_factory_ = decoder_factory; 526 } 527 528 void WebRtcVideoEngine2::SetExternalEncoderFactory( 529 WebRtcVideoEncoderFactory* encoder_factory) { 530 RTC_DCHECK(!initialized_); 531 if (external_encoder_factory_ == encoder_factory) 532 return; 533 534 // No matter what happens we shouldn't hold on to a stale 535 // WebRtcSimulcastEncoderFactory. 536 simulcast_encoder_factory_.reset(); 537 538 if (encoder_factory && 539 WebRtcSimulcastEncoderFactory::UseSimulcastEncoderFactory( 540 encoder_factory->codecs())) { 541 simulcast_encoder_factory_.reset( 542 new WebRtcSimulcastEncoderFactory(encoder_factory)); 543 encoder_factory = simulcast_encoder_factory_.get(); 544 } 545 external_encoder_factory_ = encoder_factory; 546 547 video_codecs_ = GetSupportedCodecs(); 548 } 549 550 bool WebRtcVideoEngine2::EnableTimedRender() { 551 // TODO(pbos): Figure out whether this can be removed. 552 return true; 553 } 554 555 // Checks to see whether we comprehend and could receive a particular codec 556 bool WebRtcVideoEngine2::FindCodec(const VideoCodec& in) { 557 // TODO(pbos): Probe encoder factory to figure out that the codec is supported 558 // if supported by the encoder factory. Add a corresponding test that fails 559 // with this code (that doesn't ask the factory). 560 for (size_t j = 0; j < video_codecs_.size(); ++j) { 561 VideoCodec codec(video_codecs_[j].id, video_codecs_[j].name, 0, 0, 0, 0); 562 if (codec.Matches(in)) { 563 return true; 564 } 565 } 566 return false; 567 } 568 569 // Ignore spammy trace messages, mostly from the stats API when we haven't 570 // gotten RTCP info yet from the remote side. 571 bool WebRtcVideoEngine2::ShouldIgnoreTrace(const std::string& trace) { 572 static const char* const kTracesToIgnore[] = {NULL}; 573 for (const char* const* p = kTracesToIgnore; *p; ++p) { 574 if (trace.find(*p) == 0) { 575 return true; 576 } 577 } 578 return false; 579 } 580 581 std::vector<VideoCodec> WebRtcVideoEngine2::GetSupportedCodecs() const { 582 std::vector<VideoCodec> supported_codecs = DefaultVideoCodecList(); 583 584 if (external_encoder_factory_ == NULL) { 585 return supported_codecs; 586 } 587 588 const std::vector<WebRtcVideoEncoderFactory::VideoCodec>& codecs = 589 external_encoder_factory_->codecs(); 590 for (size_t i = 0; i < codecs.size(); ++i) { 591 // Don't add internally-supported codecs twice. 592 if (CodecIsInternallySupported(codecs[i].name)) { 593 continue; 594 } 595 596 // External video encoders are given payloads 120-127. This also means that 597 // we only support up to 8 external payload types. 598 const int kExternalVideoPayloadTypeBase = 120; 599 size_t payload_type = kExternalVideoPayloadTypeBase + i; 600 RTC_DCHECK(payload_type < 128); 601 VideoCodec codec(static_cast<int>(payload_type), 602 codecs[i].name, 603 codecs[i].max_width, 604 codecs[i].max_height, 605 codecs[i].max_fps, 606 0); 607 608 AddDefaultFeedbackParams(&codec); 609 supported_codecs.push_back(codec); 610 } 611 return supported_codecs; 612 } 613 614 WebRtcVideoChannel2::WebRtcVideoChannel2( 615 webrtc::Call* call, 616 const VideoOptions& options, 617 const std::vector<VideoCodec>& recv_codecs, 618 WebRtcVideoEncoderFactory* external_encoder_factory, 619 WebRtcVideoDecoderFactory* external_decoder_factory) 620 : call_(call), 621 unsignalled_ssrc_handler_(&default_unsignalled_ssrc_handler_), 622 external_encoder_factory_(external_encoder_factory), 623 external_decoder_factory_(external_decoder_factory) { 624 RTC_DCHECK(thread_checker_.CalledOnValidThread()); 625 SetDefaultOptions(); 626 options_.SetAll(options); 627 if (options_.cpu_overuse_detection) 628 signal_cpu_adaptation_ = *options_.cpu_overuse_detection; 629 rtcp_receiver_report_ssrc_ = kDefaultRtcpReceiverReportSsrc; 630 sending_ = false; 631 default_send_ssrc_ = 0; 632 SetRecvCodecs(recv_codecs); 633 } 634 635 void WebRtcVideoChannel2::SetDefaultOptions() { 636 options_.cpu_overuse_detection = rtc::Optional<bool>(true); 637 options_.dscp = rtc::Optional<bool>(false); 638 options_.suspend_below_min_bitrate = rtc::Optional<bool>(false); 639 options_.screencast_min_bitrate = rtc::Optional<int>(0); 640 } 641 642 WebRtcVideoChannel2::~WebRtcVideoChannel2() { 643 for (auto& kv : send_streams_) 644 delete kv.second; 645 for (auto& kv : receive_streams_) 646 delete kv.second; 647 } 648 649 bool WebRtcVideoChannel2::CodecIsExternallySupported( 650 const std::string& name) const { 651 if (external_encoder_factory_ == NULL) { 652 return false; 653 } 654 655 const std::vector<WebRtcVideoEncoderFactory::VideoCodec> external_codecs = 656 external_encoder_factory_->codecs(); 657 for (size_t c = 0; c < external_codecs.size(); ++c) { 658 if (CodecNamesEq(name, external_codecs[c].name)) { 659 return true; 660 } 661 } 662 return false; 663 } 664 665 std::vector<WebRtcVideoChannel2::VideoCodecSettings> 666 WebRtcVideoChannel2::FilterSupportedCodecs( 667 const std::vector<WebRtcVideoChannel2::VideoCodecSettings>& mapped_codecs) 668 const { 669 std::vector<VideoCodecSettings> supported_codecs; 670 for (size_t i = 0; i < mapped_codecs.size(); ++i) { 671 const VideoCodecSettings& codec = mapped_codecs[i]; 672 if (CodecIsInternallySupported(codec.codec.name) || 673 CodecIsExternallySupported(codec.codec.name)) { 674 supported_codecs.push_back(codec); 675 } 676 } 677 return supported_codecs; 678 } 679 680 bool WebRtcVideoChannel2::ReceiveCodecsHaveChanged( 681 std::vector<VideoCodecSettings> before, 682 std::vector<VideoCodecSettings> after) { 683 if (before.size() != after.size()) { 684 return true; 685 } 686 // The receive codec order doesn't matter, so we sort the codecs before 687 // comparing. This is necessary because currently the 688 // only way to change the send codec is to munge SDP, which causes 689 // the receive codec list to change order, which causes the streams 690 // to be recreates which causes a "blink" of black video. In order 691 // to support munging the SDP in this way without recreating receive 692 // streams, we ignore the order of the received codecs so that 693 // changing the order doesn't cause this "blink". 694 auto comparison = 695 [](const VideoCodecSettings& codec1, const VideoCodecSettings& codec2) { 696 return codec1.codec.id > codec2.codec.id; 697 }; 698 std::sort(before.begin(), before.end(), comparison); 699 std::sort(after.begin(), after.end(), comparison); 700 for (size_t i = 0; i < before.size(); ++i) { 701 // For the same reason that we sort the codecs, we also ignore the 702 // preference. We don't want a preference change on the receive 703 // side to cause recreation of the stream. 704 before[i].codec.preference = 0; 705 after[i].codec.preference = 0; 706 if (before[i] != after[i]) { 707 return true; 708 } 709 } 710 return false; 711 } 712 713 bool WebRtcVideoChannel2::SetSendParameters(const VideoSendParameters& params) { 714 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetSendParameters"); 715 LOG(LS_INFO) << "SetSendParameters: " << params.ToString(); 716 // TODO(pbos): Refactor this to only recreate the send streams once 717 // instead of 4 times. 718 if (!SetSendCodecs(params.codecs) || 719 !SetSendRtpHeaderExtensions(params.extensions) || 720 !SetMaxSendBandwidth(params.max_bandwidth_bps) || 721 !SetOptions(params.options)) { 722 return false; 723 } 724 if (send_params_.rtcp.reduced_size != params.rtcp.reduced_size) { 725 rtc::CritScope stream_lock(&stream_crit_); 726 for (auto& kv : send_streams_) { 727 kv.second->SetSendParameters(params); 728 } 729 } 730 send_params_ = params; 731 return true; 732 } 733 734 bool WebRtcVideoChannel2::SetRecvParameters(const VideoRecvParameters& params) { 735 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetRecvParameters"); 736 LOG(LS_INFO) << "SetRecvParameters: " << params.ToString(); 737 // TODO(pbos): Refactor this to only recreate the recv streams once 738 // instead of twice. 739 if (!SetRecvCodecs(params.codecs) || 740 !SetRecvRtpHeaderExtensions(params.extensions)) { 741 return false; 742 } 743 if (recv_params_.rtcp.reduced_size != params.rtcp.reduced_size) { 744 rtc::CritScope stream_lock(&stream_crit_); 745 for (auto& kv : receive_streams_) { 746 kv.second->SetRecvParameters(params); 747 } 748 } 749 recv_params_ = params; 750 return true; 751 } 752 753 std::string WebRtcVideoChannel2::CodecSettingsVectorToString( 754 const std::vector<VideoCodecSettings>& codecs) { 755 std::stringstream out; 756 out << '{'; 757 for (size_t i = 0; i < codecs.size(); ++i) { 758 out << codecs[i].codec.ToString(); 759 if (i != codecs.size() - 1) { 760 out << ", "; 761 } 762 } 763 out << '}'; 764 return out.str(); 765 } 766 767 bool WebRtcVideoChannel2::SetRecvCodecs(const std::vector<VideoCodec>& codecs) { 768 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetRecvCodecs"); 769 LOG(LS_INFO) << "SetRecvCodecs: " << CodecVectorToString(codecs); 770 if (!ValidateCodecFormats(codecs)) { 771 return false; 772 } 773 774 const std::vector<VideoCodecSettings> mapped_codecs = MapCodecs(codecs); 775 if (mapped_codecs.empty()) { 776 LOG(LS_ERROR) << "SetRecvCodecs called without any video codecs."; 777 return false; 778 } 779 780 std::vector<VideoCodecSettings> supported_codecs = 781 FilterSupportedCodecs(mapped_codecs); 782 783 if (mapped_codecs.size() != supported_codecs.size()) { 784 LOG(LS_ERROR) << "SetRecvCodecs called with unsupported video codecs."; 785 return false; 786 } 787 788 // Prevent reconfiguration when setting identical receive codecs. 789 if (!ReceiveCodecsHaveChanged(recv_codecs_, supported_codecs)) { 790 LOG(LS_INFO) 791 << "Ignoring call to SetRecvCodecs because codecs haven't changed."; 792 return true; 793 } 794 795 LOG(LS_INFO) << "Changing recv codecs from " 796 << CodecSettingsVectorToString(recv_codecs_) << " to " 797 << CodecSettingsVectorToString(supported_codecs); 798 recv_codecs_ = supported_codecs; 799 800 rtc::CritScope stream_lock(&stream_crit_); 801 for (std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it = 802 receive_streams_.begin(); 803 it != receive_streams_.end(); ++it) { 804 it->second->SetRecvCodecs(recv_codecs_); 805 } 806 807 return true; 808 } 809 810 bool WebRtcVideoChannel2::SetSendCodecs(const std::vector<VideoCodec>& codecs) { 811 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetSendCodecs"); 812 LOG(LS_INFO) << "SetSendCodecs: " << CodecVectorToString(codecs); 813 if (!ValidateCodecFormats(codecs)) { 814 return false; 815 } 816 817 const std::vector<VideoCodecSettings> supported_codecs = 818 FilterSupportedCodecs(MapCodecs(codecs)); 819 820 if (supported_codecs.empty()) { 821 LOG(LS_ERROR) << "No video codecs supported."; 822 return false; 823 } 824 825 LOG(LS_INFO) << "Using codec: " << supported_codecs.front().codec.ToString(); 826 827 if (send_codec_ && supported_codecs.front() == *send_codec_) { 828 LOG(LS_INFO) << "Ignore call to SetSendCodecs because first supported " 829 "codec hasn't changed."; 830 // Using same codec, avoid reconfiguring. 831 return true; 832 } 833 834 send_codec_ = rtc::Optional<WebRtcVideoChannel2::VideoCodecSettings>( 835 supported_codecs.front()); 836 837 rtc::CritScope stream_lock(&stream_crit_); 838 LOG(LS_INFO) << "Change the send codec because SetSendCodecs has a different " 839 "first supported codec."; 840 for (auto& kv : send_streams_) { 841 RTC_DCHECK(kv.second != nullptr); 842 kv.second->SetCodec(supported_codecs.front()); 843 } 844 LOG(LS_INFO) 845 << "SetFeedbackOptions on all the receive streams because the send " 846 "codec has changed."; 847 for (auto& kv : receive_streams_) { 848 RTC_DCHECK(kv.second != nullptr); 849 kv.second->SetFeedbackParameters( 850 HasNack(supported_codecs.front().codec), 851 HasRemb(supported_codecs.front().codec), 852 HasTransportCc(supported_codecs.front().codec)); 853 } 854 855 // TODO(holmer): Changing the codec parameters shouldn't necessarily mean that 856 // we change the min/max of bandwidth estimation. Reevaluate this. 857 VideoCodec codec = supported_codecs.front().codec; 858 int bitrate_kbps; 859 if (codec.GetParam(kCodecParamMinBitrate, &bitrate_kbps) && 860 bitrate_kbps > 0) { 861 bitrate_config_.min_bitrate_bps = bitrate_kbps * 1000; 862 } else { 863 bitrate_config_.min_bitrate_bps = 0; 864 } 865 if (codec.GetParam(kCodecParamStartBitrate, &bitrate_kbps) && 866 bitrate_kbps > 0) { 867 bitrate_config_.start_bitrate_bps = bitrate_kbps * 1000; 868 } else { 869 // Do not reconfigure start bitrate unless it's specified and positive. 870 bitrate_config_.start_bitrate_bps = -1; 871 } 872 if (codec.GetParam(kCodecParamMaxBitrate, &bitrate_kbps) && 873 bitrate_kbps > 0) { 874 bitrate_config_.max_bitrate_bps = bitrate_kbps * 1000; 875 } else { 876 bitrate_config_.max_bitrate_bps = -1; 877 } 878 call_->SetBitrateConfig(bitrate_config_); 879 880 return true; 881 } 882 883 bool WebRtcVideoChannel2::GetSendCodec(VideoCodec* codec) { 884 if (!send_codec_) { 885 LOG(LS_VERBOSE) << "GetSendCodec: No send codec set."; 886 return false; 887 } 888 *codec = send_codec_->codec; 889 return true; 890 } 891 892 bool WebRtcVideoChannel2::SetSendStreamFormat(uint32_t ssrc, 893 const VideoFormat& format) { 894 LOG(LS_VERBOSE) << "SetSendStreamFormat:" << ssrc << " -> " 895 << format.ToString(); 896 rtc::CritScope stream_lock(&stream_crit_); 897 if (send_streams_.find(ssrc) == send_streams_.end()) { 898 return false; 899 } 900 return send_streams_[ssrc]->SetVideoFormat(format); 901 } 902 903 bool WebRtcVideoChannel2::SetSend(bool send) { 904 LOG(LS_VERBOSE) << "SetSend: " << (send ? "true" : "false"); 905 if (send && !send_codec_) { 906 LOG(LS_ERROR) << "SetSend(true) called before setting codec."; 907 return false; 908 } 909 if (send) { 910 StartAllSendStreams(); 911 } else { 912 StopAllSendStreams(); 913 } 914 sending_ = send; 915 return true; 916 } 917 918 bool WebRtcVideoChannel2::SetVideoSend(uint32_t ssrc, bool enable, 919 const VideoOptions* options) { 920 // TODO(solenberg): The state change should be fully rolled back if any one of 921 // these calls fail. 922 if (!MuteStream(ssrc, !enable)) { 923 return false; 924 } 925 if (enable && options) { 926 return SetOptions(*options); 927 } else { 928 return true; 929 } 930 } 931 932 bool WebRtcVideoChannel2::ValidateSendSsrcAvailability( 933 const StreamParams& sp) const { 934 for (uint32_t ssrc: sp.ssrcs) { 935 if (send_ssrcs_.find(ssrc) != send_ssrcs_.end()) { 936 LOG(LS_ERROR) << "Send stream with SSRC '" << ssrc << "' already exists."; 937 return false; 938 } 939 } 940 return true; 941 } 942 943 bool WebRtcVideoChannel2::ValidateReceiveSsrcAvailability( 944 const StreamParams& sp) const { 945 for (uint32_t ssrc: sp.ssrcs) { 946 if (receive_ssrcs_.find(ssrc) != receive_ssrcs_.end()) { 947 LOG(LS_ERROR) << "Receive stream with SSRC '" << ssrc 948 << "' already exists."; 949 return false; 950 } 951 } 952 return true; 953 } 954 955 bool WebRtcVideoChannel2::AddSendStream(const StreamParams& sp) { 956 LOG(LS_INFO) << "AddSendStream: " << sp.ToString(); 957 if (!ValidateStreamParams(sp)) 958 return false; 959 960 rtc::CritScope stream_lock(&stream_crit_); 961 962 if (!ValidateSendSsrcAvailability(sp)) 963 return false; 964 965 for (uint32_t used_ssrc : sp.ssrcs) 966 send_ssrcs_.insert(used_ssrc); 967 968 webrtc::VideoSendStream::Config config(this); 969 config.overuse_callback = this; 970 971 WebRtcVideoSendStream* stream = new WebRtcVideoSendStream( 972 call_, sp, config, external_encoder_factory_, options_, 973 bitrate_config_.max_bitrate_bps, send_codec_, send_rtp_extensions_, 974 send_params_); 975 976 uint32_t ssrc = sp.first_ssrc(); 977 RTC_DCHECK(ssrc != 0); 978 send_streams_[ssrc] = stream; 979 980 if (rtcp_receiver_report_ssrc_ == kDefaultRtcpReceiverReportSsrc) { 981 rtcp_receiver_report_ssrc_ = ssrc; 982 LOG(LS_INFO) << "SetLocalSsrc on all the receive streams because we added " 983 "a send stream."; 984 for (auto& kv : receive_streams_) 985 kv.second->SetLocalSsrc(ssrc); 986 } 987 if (default_send_ssrc_ == 0) { 988 default_send_ssrc_ = ssrc; 989 } 990 if (sending_) { 991 stream->Start(); 992 } 993 994 return true; 995 } 996 997 bool WebRtcVideoChannel2::RemoveSendStream(uint32_t ssrc) { 998 LOG(LS_INFO) << "RemoveSendStream: " << ssrc; 999 1000 if (ssrc == 0) { 1001 if (default_send_ssrc_ == 0) { 1002 LOG(LS_ERROR) << "No default send stream active."; 1003 return false; 1004 } 1005 1006 LOG(LS_VERBOSE) << "Removing default stream: " << default_send_ssrc_; 1007 ssrc = default_send_ssrc_; 1008 } 1009 1010 WebRtcVideoSendStream* removed_stream; 1011 { 1012 rtc::CritScope stream_lock(&stream_crit_); 1013 std::map<uint32_t, WebRtcVideoSendStream*>::iterator it = 1014 send_streams_.find(ssrc); 1015 if (it == send_streams_.end()) { 1016 return false; 1017 } 1018 1019 for (uint32_t old_ssrc : it->second->GetSsrcs()) 1020 send_ssrcs_.erase(old_ssrc); 1021 1022 removed_stream = it->second; 1023 send_streams_.erase(it); 1024 1025 // Switch receiver report SSRCs, the one in use is no longer valid. 1026 if (rtcp_receiver_report_ssrc_ == ssrc) { 1027 rtcp_receiver_report_ssrc_ = send_streams_.empty() 1028 ? kDefaultRtcpReceiverReportSsrc 1029 : send_streams_.begin()->first; 1030 LOG(LS_INFO) << "SetLocalSsrc on all the receive streams because the " 1031 "previous local SSRC was removed."; 1032 1033 for (auto& kv : receive_streams_) { 1034 kv.second->SetLocalSsrc(rtcp_receiver_report_ssrc_); 1035 } 1036 } 1037 } 1038 1039 delete removed_stream; 1040 1041 if (ssrc == default_send_ssrc_) { 1042 default_send_ssrc_ = 0; 1043 } 1044 1045 return true; 1046 } 1047 1048 void WebRtcVideoChannel2::DeleteReceiveStream( 1049 WebRtcVideoChannel2::WebRtcVideoReceiveStream* stream) { 1050 for (uint32_t old_ssrc : stream->GetSsrcs()) 1051 receive_ssrcs_.erase(old_ssrc); 1052 delete stream; 1053 } 1054 1055 bool WebRtcVideoChannel2::AddRecvStream(const StreamParams& sp) { 1056 return AddRecvStream(sp, false); 1057 } 1058 1059 bool WebRtcVideoChannel2::AddRecvStream(const StreamParams& sp, 1060 bool default_stream) { 1061 RTC_DCHECK(thread_checker_.CalledOnValidThread()); 1062 1063 LOG(LS_INFO) << "AddRecvStream" << (default_stream ? " (default stream)" : "") 1064 << ": " << sp.ToString(); 1065 if (!ValidateStreamParams(sp)) 1066 return false; 1067 1068 uint32_t ssrc = sp.first_ssrc(); 1069 RTC_DCHECK(ssrc != 0); // TODO(pbos): Is this ever valid? 1070 1071 rtc::CritScope stream_lock(&stream_crit_); 1072 // Remove running stream if this was a default stream. 1073 auto prev_stream = receive_streams_.find(ssrc); 1074 if (prev_stream != receive_streams_.end()) { 1075 if (default_stream || !prev_stream->second->IsDefaultStream()) { 1076 LOG(LS_ERROR) << "Receive stream for SSRC '" << ssrc 1077 << "' already exists."; 1078 return false; 1079 } 1080 DeleteReceiveStream(prev_stream->second); 1081 receive_streams_.erase(prev_stream); 1082 } 1083 1084 if (!ValidateReceiveSsrcAvailability(sp)) 1085 return false; 1086 1087 for (uint32_t used_ssrc : sp.ssrcs) 1088 receive_ssrcs_.insert(used_ssrc); 1089 1090 webrtc::VideoReceiveStream::Config config(this); 1091 ConfigureReceiverRtp(&config, sp); 1092 1093 // Set up A/V sync group based on sync label. 1094 config.sync_group = sp.sync_label; 1095 1096 config.rtp.remb = send_codec_ ? HasRemb(send_codec_->codec) : false; 1097 config.rtp.transport_cc = 1098 send_codec_ ? HasTransportCc(send_codec_->codec) : false; 1099 1100 receive_streams_[ssrc] = new WebRtcVideoReceiveStream( 1101 call_, sp, config, external_decoder_factory_, default_stream, 1102 recv_codecs_, options_.disable_prerenderer_smoothing.value_or(false)); 1103 1104 return true; 1105 } 1106 1107 void WebRtcVideoChannel2::ConfigureReceiverRtp( 1108 webrtc::VideoReceiveStream::Config* config, 1109 const StreamParams& sp) const { 1110 uint32_t ssrc = sp.first_ssrc(); 1111 1112 config->rtp.remote_ssrc = ssrc; 1113 config->rtp.local_ssrc = rtcp_receiver_report_ssrc_; 1114 1115 config->rtp.extensions = recv_rtp_extensions_; 1116 config->rtp.rtcp_mode = recv_params_.rtcp.reduced_size 1117 ? webrtc::RtcpMode::kReducedSize 1118 : webrtc::RtcpMode::kCompound; 1119 1120 // TODO(pbos): This protection is against setting the same local ssrc as 1121 // remote which is not permitted by the lower-level API. RTCP requires a 1122 // corresponding sender SSRC. Figure out what to do when we don't have 1123 // (receive-only) or know a good local SSRC. 1124 if (config->rtp.remote_ssrc == config->rtp.local_ssrc) { 1125 if (config->rtp.local_ssrc != kDefaultRtcpReceiverReportSsrc) { 1126 config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc; 1127 } else { 1128 config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc + 1; 1129 } 1130 } 1131 1132 for (size_t i = 0; i < recv_codecs_.size(); ++i) { 1133 MergeFecConfig(recv_codecs_[i].fec, &config->rtp.fec); 1134 } 1135 1136 for (size_t i = 0; i < recv_codecs_.size(); ++i) { 1137 uint32_t rtx_ssrc; 1138 if (recv_codecs_[i].rtx_payload_type != -1 && 1139 sp.GetFidSsrc(ssrc, &rtx_ssrc)) { 1140 webrtc::VideoReceiveStream::Config::Rtp::Rtx& rtx = 1141 config->rtp.rtx[recv_codecs_[i].codec.id]; 1142 rtx.ssrc = rtx_ssrc; 1143 rtx.payload_type = recv_codecs_[i].rtx_payload_type; 1144 } 1145 } 1146 } 1147 1148 bool WebRtcVideoChannel2::RemoveRecvStream(uint32_t ssrc) { 1149 LOG(LS_INFO) << "RemoveRecvStream: " << ssrc; 1150 if (ssrc == 0) { 1151 LOG(LS_ERROR) << "RemoveRecvStream with 0 ssrc is not supported."; 1152 return false; 1153 } 1154 1155 rtc::CritScope stream_lock(&stream_crit_); 1156 std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator stream = 1157 receive_streams_.find(ssrc); 1158 if (stream == receive_streams_.end()) { 1159 LOG(LS_ERROR) << "Stream not found for ssrc: " << ssrc; 1160 return false; 1161 } 1162 DeleteReceiveStream(stream->second); 1163 receive_streams_.erase(stream); 1164 1165 return true; 1166 } 1167 1168 bool WebRtcVideoChannel2::SetRenderer(uint32_t ssrc, VideoRenderer* renderer) { 1169 LOG(LS_INFO) << "SetRenderer: ssrc:" << ssrc << " " 1170 << (renderer ? "(ptr)" : "NULL"); 1171 if (ssrc == 0) { 1172 default_unsignalled_ssrc_handler_.SetDefaultRenderer(this, renderer); 1173 return true; 1174 } 1175 1176 rtc::CritScope stream_lock(&stream_crit_); 1177 std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it = 1178 receive_streams_.find(ssrc); 1179 if (it == receive_streams_.end()) { 1180 return false; 1181 } 1182 1183 it->second->SetRenderer(renderer); 1184 return true; 1185 } 1186 1187 bool WebRtcVideoChannel2::GetRenderer(uint32_t ssrc, VideoRenderer** renderer) { 1188 if (ssrc == 0) { 1189 *renderer = default_unsignalled_ssrc_handler_.GetDefaultRenderer(); 1190 return *renderer != NULL; 1191 } 1192 1193 rtc::CritScope stream_lock(&stream_crit_); 1194 std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it = 1195 receive_streams_.find(ssrc); 1196 if (it == receive_streams_.end()) { 1197 return false; 1198 } 1199 *renderer = it->second->GetRenderer(); 1200 return true; 1201 } 1202 1203 bool WebRtcVideoChannel2::GetStats(VideoMediaInfo* info) { 1204 info->Clear(); 1205 FillSenderStats(info); 1206 FillReceiverStats(info); 1207 webrtc::Call::Stats stats = call_->GetStats(); 1208 FillBandwidthEstimationStats(stats, info); 1209 if (stats.rtt_ms != -1) { 1210 for (size_t i = 0; i < info->senders.size(); ++i) { 1211 info->senders[i].rtt_ms = stats.rtt_ms; 1212 } 1213 } 1214 return true; 1215 } 1216 1217 void WebRtcVideoChannel2::FillSenderStats(VideoMediaInfo* video_media_info) { 1218 rtc::CritScope stream_lock(&stream_crit_); 1219 for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator it = 1220 send_streams_.begin(); 1221 it != send_streams_.end(); ++it) { 1222 video_media_info->senders.push_back(it->second->GetVideoSenderInfo()); 1223 } 1224 } 1225 1226 void WebRtcVideoChannel2::FillReceiverStats(VideoMediaInfo* video_media_info) { 1227 rtc::CritScope stream_lock(&stream_crit_); 1228 for (std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it = 1229 receive_streams_.begin(); 1230 it != receive_streams_.end(); ++it) { 1231 video_media_info->receivers.push_back(it->second->GetVideoReceiverInfo()); 1232 } 1233 } 1234 1235 void WebRtcVideoChannel2::FillBandwidthEstimationStats( 1236 const webrtc::Call::Stats& stats, 1237 VideoMediaInfo* video_media_info) { 1238 BandwidthEstimationInfo bwe_info; 1239 bwe_info.available_send_bandwidth = stats.send_bandwidth_bps; 1240 bwe_info.available_recv_bandwidth = stats.recv_bandwidth_bps; 1241 bwe_info.bucket_delay = stats.pacer_delay_ms; 1242 1243 // Get send stream bitrate stats. 1244 rtc::CritScope stream_lock(&stream_crit_); 1245 for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator stream = 1246 send_streams_.begin(); 1247 stream != send_streams_.end(); ++stream) { 1248 stream->second->FillBandwidthEstimationInfo(&bwe_info); 1249 } 1250 video_media_info->bw_estimations.push_back(bwe_info); 1251 } 1252 1253 bool WebRtcVideoChannel2::SetCapturer(uint32_t ssrc, VideoCapturer* capturer) { 1254 LOG(LS_INFO) << "SetCapturer: " << ssrc << " -> " 1255 << (capturer != NULL ? "(capturer)" : "NULL"); 1256 RTC_DCHECK(ssrc != 0); 1257 { 1258 rtc::CritScope stream_lock(&stream_crit_); 1259 if (send_streams_.find(ssrc) == send_streams_.end()) { 1260 LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc; 1261 return false; 1262 } 1263 if (!send_streams_[ssrc]->SetCapturer(capturer)) { 1264 return false; 1265 } 1266 } 1267 1268 if (capturer) { 1269 capturer->SetApplyRotation( 1270 !FindHeaderExtension(send_rtp_extensions_, 1271 kRtpVideoRotationHeaderExtension)); 1272 } 1273 { 1274 rtc::CritScope lock(&capturer_crit_); 1275 capturers_[ssrc] = capturer; 1276 } 1277 return true; 1278 } 1279 1280 bool WebRtcVideoChannel2::SendIntraFrame() { 1281 // TODO(pbos): Implement. 1282 LOG(LS_VERBOSE) << "SendIntraFrame()."; 1283 return true; 1284 } 1285 1286 bool WebRtcVideoChannel2::RequestIntraFrame() { 1287 // TODO(pbos): Implement. 1288 LOG(LS_VERBOSE) << "SendIntraFrame()."; 1289 return true; 1290 } 1291 1292 void WebRtcVideoChannel2::OnPacketReceived( 1293 rtc::Buffer* packet, 1294 const rtc::PacketTime& packet_time) { 1295 const webrtc::PacketTime webrtc_packet_time(packet_time.timestamp, 1296 packet_time.not_before); 1297 const webrtc::PacketReceiver::DeliveryStatus delivery_result = 1298 call_->Receiver()->DeliverPacket( 1299 webrtc::MediaType::VIDEO, 1300 reinterpret_cast<const uint8_t*>(packet->data()), packet->size(), 1301 webrtc_packet_time); 1302 switch (delivery_result) { 1303 case webrtc::PacketReceiver::DELIVERY_OK: 1304 return; 1305 case webrtc::PacketReceiver::DELIVERY_PACKET_ERROR: 1306 return; 1307 case webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC: 1308 break; 1309 } 1310 1311 uint32_t ssrc = 0; 1312 if (!GetRtpSsrc(packet->data(), packet->size(), &ssrc)) { 1313 return; 1314 } 1315 1316 int payload_type = 0; 1317 if (!GetRtpPayloadType(packet->data(), packet->size(), &payload_type)) { 1318 return; 1319 } 1320 1321 // See if this payload_type is registered as one that usually gets its own 1322 // SSRC (RTX) or at least is safe to drop either way (ULPFEC). If it is, and 1323 // it wasn't handled above by DeliverPacket, that means we don't know what 1324 // stream it associates with, and we shouldn't ever create an implicit channel 1325 // for these. 1326 for (auto& codec : recv_codecs_) { 1327 if (payload_type == codec.rtx_payload_type || 1328 payload_type == codec.fec.red_rtx_payload_type || 1329 payload_type == codec.fec.ulpfec_payload_type) { 1330 return; 1331 } 1332 } 1333 1334 switch (unsignalled_ssrc_handler_->OnUnsignalledSsrc(this, ssrc)) { 1335 case UnsignalledSsrcHandler::kDropPacket: 1336 return; 1337 case UnsignalledSsrcHandler::kDeliverPacket: 1338 break; 1339 } 1340 1341 if (call_->Receiver()->DeliverPacket( 1342 webrtc::MediaType::VIDEO, 1343 reinterpret_cast<const uint8_t*>(packet->data()), packet->size(), 1344 webrtc_packet_time) != webrtc::PacketReceiver::DELIVERY_OK) { 1345 LOG(LS_WARNING) << "Failed to deliver RTP packet on re-delivery."; 1346 return; 1347 } 1348 } 1349 1350 void WebRtcVideoChannel2::OnRtcpReceived( 1351 rtc::Buffer* packet, 1352 const rtc::PacketTime& packet_time) { 1353 const webrtc::PacketTime webrtc_packet_time(packet_time.timestamp, 1354 packet_time.not_before); 1355 // TODO(pbos): Check webrtc::PacketReceiver::DELIVERY_OK once we deliver 1356 // for both audio and video on the same path. Since BundleFilter doesn't 1357 // filter RTCP anymore incoming RTCP packets could've been going to audio (so 1358 // logging failures spam the log). 1359 call_->Receiver()->DeliverPacket( 1360 webrtc::MediaType::VIDEO, 1361 reinterpret_cast<const uint8_t*>(packet->data()), packet->size(), 1362 webrtc_packet_time); 1363 } 1364 1365 void WebRtcVideoChannel2::OnReadyToSend(bool ready) { 1366 LOG(LS_VERBOSE) << "OnReadyToSend: " << (ready ? "Ready." : "Not ready."); 1367 call_->SignalNetworkState(ready ? webrtc::kNetworkUp : webrtc::kNetworkDown); 1368 } 1369 1370 bool WebRtcVideoChannel2::MuteStream(uint32_t ssrc, bool mute) { 1371 LOG(LS_VERBOSE) << "MuteStream: " << ssrc << " -> " 1372 << (mute ? "mute" : "unmute"); 1373 RTC_DCHECK(ssrc != 0); 1374 rtc::CritScope stream_lock(&stream_crit_); 1375 if (send_streams_.find(ssrc) == send_streams_.end()) { 1376 LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc; 1377 return false; 1378 } 1379 1380 send_streams_[ssrc]->MuteStream(mute); 1381 return true; 1382 } 1383 1384 bool WebRtcVideoChannel2::SetRecvRtpHeaderExtensions( 1385 const std::vector<RtpHeaderExtension>& extensions) { 1386 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetRecvRtpHeaderExtensions"); 1387 if (!ValidateRtpExtensions(extensions)) { 1388 return false; 1389 } 1390 std::vector<webrtc::RtpExtension> filtered_extensions = FilterRtpExtensions( 1391 extensions, webrtc::RtpExtension::IsSupportedForVideo, false); 1392 if (recv_rtp_extensions_ == filtered_extensions) { 1393 LOG(LS_INFO) << "Ignoring call to SetRecvRtpHeaderExtensions because " 1394 "header extensions haven't changed."; 1395 return true; 1396 } 1397 recv_rtp_extensions_.swap(filtered_extensions); 1398 1399 rtc::CritScope stream_lock(&stream_crit_); 1400 for (std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it = 1401 receive_streams_.begin(); 1402 it != receive_streams_.end(); ++it) { 1403 it->second->SetRtpExtensions(recv_rtp_extensions_); 1404 } 1405 return true; 1406 } 1407 1408 bool WebRtcVideoChannel2::SetSendRtpHeaderExtensions( 1409 const std::vector<RtpHeaderExtension>& extensions) { 1410 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetSendRtpHeaderExtensions"); 1411 if (!ValidateRtpExtensions(extensions)) { 1412 return false; 1413 } 1414 std::vector<webrtc::RtpExtension> filtered_extensions = FilterRtpExtensions( 1415 extensions, webrtc::RtpExtension::IsSupportedForVideo, true); 1416 if (send_rtp_extensions_ == filtered_extensions) { 1417 LOG(LS_INFO) << "Ignoring call to SetRecvRtpHeaderExtensions because " 1418 "header extensions haven't changed."; 1419 return true; 1420 } 1421 send_rtp_extensions_.swap(filtered_extensions); 1422 1423 const webrtc::RtpExtension* cvo_extension = FindHeaderExtension( 1424 send_rtp_extensions_, kRtpVideoRotationHeaderExtension); 1425 1426 rtc::CritScope stream_lock(&stream_crit_); 1427 for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator it = 1428 send_streams_.begin(); 1429 it != send_streams_.end(); ++it) { 1430 it->second->SetRtpExtensions(send_rtp_extensions_); 1431 it->second->SetApplyRotation(!cvo_extension); 1432 } 1433 return true; 1434 } 1435 1436 // Counter-intuitively this method doesn't only set global bitrate caps but also 1437 // per-stream codec max bitrates. This is to permit SetMaxSendBitrate (b=AS) to 1438 // raise bitrates above the 2000k default bitrate cap. 1439 bool WebRtcVideoChannel2::SetMaxSendBandwidth(int max_bitrate_bps) { 1440 // TODO(pbos): Figure out whether b=AS means max bitrate for this 1441 // WebRtcVideoChannel2 (in which case we're good), or per sender (SSRC), in 1442 // which case this should not set a Call::BitrateConfig but rather reconfigure 1443 // all senders. 1444 LOG(LS_INFO) << "SetMaxSendBandwidth: " << max_bitrate_bps << "bps."; 1445 if (max_bitrate_bps == bitrate_config_.max_bitrate_bps) 1446 return true; 1447 1448 if (max_bitrate_bps < 0) { 1449 // Option not set. 1450 return true; 1451 } 1452 if (max_bitrate_bps == 0) { 1453 // Unsetting max bitrate. 1454 max_bitrate_bps = -1; 1455 } 1456 bitrate_config_.start_bitrate_bps = -1; 1457 bitrate_config_.max_bitrate_bps = max_bitrate_bps; 1458 if (max_bitrate_bps > 0 && 1459 bitrate_config_.min_bitrate_bps > max_bitrate_bps) { 1460 bitrate_config_.min_bitrate_bps = max_bitrate_bps; 1461 } 1462 call_->SetBitrateConfig(bitrate_config_); 1463 rtc::CritScope stream_lock(&stream_crit_); 1464 for (auto& kv : send_streams_) 1465 kv.second->SetMaxBitrateBps(max_bitrate_bps); 1466 return true; 1467 } 1468 1469 bool WebRtcVideoChannel2::SetOptions(const VideoOptions& options) { 1470 TRACE_EVENT0("webrtc", "WebRtcVideoChannel2::SetOptions"); 1471 LOG(LS_INFO) << "SetOptions: " << options.ToString(); 1472 VideoOptions old_options = options_; 1473 options_.SetAll(options); 1474 if (options_ == old_options) { 1475 // No new options to set. 1476 return true; 1477 } 1478 { 1479 rtc::CritScope lock(&capturer_crit_); 1480 if (options_.cpu_overuse_detection) 1481 signal_cpu_adaptation_ = *options_.cpu_overuse_detection; 1482 } 1483 rtc::DiffServCodePoint dscp = 1484 options_.dscp.value_or(false) ? rtc::DSCP_AF41 : rtc::DSCP_DEFAULT; 1485 MediaChannel::SetDscp(dscp); 1486 rtc::CritScope stream_lock(&stream_crit_); 1487 for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator it = 1488 send_streams_.begin(); 1489 it != send_streams_.end(); ++it) { 1490 it->second->SetOptions(options_); 1491 } 1492 return true; 1493 } 1494 1495 void WebRtcVideoChannel2::SetInterface(NetworkInterface* iface) { 1496 MediaChannel::SetInterface(iface); 1497 // Set the RTP recv/send buffer to a bigger size 1498 MediaChannel::SetOption(NetworkInterface::ST_RTP, 1499 rtc::Socket::OPT_RCVBUF, 1500 kVideoRtpBufferSize); 1501 1502 // Speculative change to increase the outbound socket buffer size. 1503 // In b/15152257, we are seeing a significant number of packets discarded 1504 // due to lack of socket buffer space, although it's not yet clear what the 1505 // ideal value should be. 1506 MediaChannel::SetOption(NetworkInterface::ST_RTP, 1507 rtc::Socket::OPT_SNDBUF, 1508 kVideoRtpBufferSize); 1509 } 1510 1511 void WebRtcVideoChannel2::UpdateAspectRatio(int ratio_w, int ratio_h) { 1512 // TODO(pbos): Implement. 1513 } 1514 1515 void WebRtcVideoChannel2::OnMessage(rtc::Message* msg) { 1516 // Ignored. 1517 } 1518 1519 void WebRtcVideoChannel2::OnLoadUpdate(Load load) { 1520 // OnLoadUpdate can not take any locks that are held while creating streams 1521 // etc. Doing so establishes lock-order inversions between the webrtc process 1522 // thread on stream creation and locks such as stream_crit_ while calling out. 1523 rtc::CritScope stream_lock(&capturer_crit_); 1524 if (!signal_cpu_adaptation_) 1525 return; 1526 // Do not adapt resolution for screen content as this will likely result in 1527 // blurry and unreadable text. 1528 for (auto& kv : capturers_) { 1529 if (kv.second != nullptr 1530 && !kv.second->IsScreencast() 1531 && kv.second->video_adapter() != nullptr) { 1532 kv.second->video_adapter()->OnCpuResolutionRequest( 1533 load == kOveruse ? CoordinatedVideoAdapter::DOWNGRADE 1534 : CoordinatedVideoAdapter::UPGRADE); 1535 } 1536 } 1537 } 1538 1539 bool WebRtcVideoChannel2::SendRtp(const uint8_t* data, 1540 size_t len, 1541 const webrtc::PacketOptions& options) { 1542 rtc::Buffer packet(data, len, kMaxRtpPacketLen); 1543 rtc::PacketOptions rtc_options; 1544 rtc_options.packet_id = options.packet_id; 1545 return MediaChannel::SendPacket(&packet, rtc_options); 1546 } 1547 1548 bool WebRtcVideoChannel2::SendRtcp(const uint8_t* data, size_t len) { 1549 rtc::Buffer packet(data, len, kMaxRtpPacketLen); 1550 return MediaChannel::SendRtcp(&packet, rtc::PacketOptions()); 1551 } 1552 1553 void WebRtcVideoChannel2::StartAllSendStreams() { 1554 rtc::CritScope stream_lock(&stream_crit_); 1555 for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator it = 1556 send_streams_.begin(); 1557 it != send_streams_.end(); ++it) { 1558 it->second->Start(); 1559 } 1560 } 1561 1562 void WebRtcVideoChannel2::StopAllSendStreams() { 1563 rtc::CritScope stream_lock(&stream_crit_); 1564 for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator it = 1565 send_streams_.begin(); 1566 it != send_streams_.end(); ++it) { 1567 it->second->Stop(); 1568 } 1569 } 1570 1571 WebRtcVideoChannel2::WebRtcVideoSendStream::VideoSendStreamParameters:: 1572 VideoSendStreamParameters( 1573 const webrtc::VideoSendStream::Config& config, 1574 const VideoOptions& options, 1575 int max_bitrate_bps, 1576 const rtc::Optional<VideoCodecSettings>& codec_settings) 1577 : config(config), 1578 options(options), 1579 max_bitrate_bps(max_bitrate_bps), 1580 codec_settings(codec_settings) {} 1581 1582 WebRtcVideoChannel2::WebRtcVideoSendStream::AllocatedEncoder::AllocatedEncoder( 1583 webrtc::VideoEncoder* encoder, 1584 webrtc::VideoCodecType type, 1585 bool external) 1586 : encoder(encoder), 1587 external_encoder(nullptr), 1588 type(type), 1589 external(external) { 1590 if (external) { 1591 external_encoder = encoder; 1592 this->encoder = 1593 new webrtc::VideoEncoderSoftwareFallbackWrapper(type, encoder); 1594 } 1595 } 1596 1597 WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream( 1598 webrtc::Call* call, 1599 const StreamParams& sp, 1600 const webrtc::VideoSendStream::Config& config, 1601 WebRtcVideoEncoderFactory* external_encoder_factory, 1602 const VideoOptions& options, 1603 int max_bitrate_bps, 1604 const rtc::Optional<VideoCodecSettings>& codec_settings, 1605 const std::vector<webrtc::RtpExtension>& rtp_extensions, 1606 // TODO(deadbeef): Don't duplicate information between send_params, 1607 // rtp_extensions, options, etc. 1608 const VideoSendParameters& send_params) 1609 : ssrcs_(sp.ssrcs), 1610 ssrc_groups_(sp.ssrc_groups), 1611 call_(call), 1612 external_encoder_factory_(external_encoder_factory), 1613 stream_(NULL), 1614 parameters_(config, options, max_bitrate_bps, codec_settings), 1615 allocated_encoder_(NULL, webrtc::kVideoCodecUnknown, false), 1616 capturer_(NULL), 1617 sending_(false), 1618 muted_(false), 1619 old_adapt_changes_(0), 1620 first_frame_timestamp_ms_(0), 1621 last_frame_timestamp_ms_(0) { 1622 parameters_.config.rtp.max_packet_size = kVideoMtu; 1623 1624 sp.GetPrimarySsrcs(¶meters_.config.rtp.ssrcs); 1625 sp.GetFidSsrcs(parameters_.config.rtp.ssrcs, 1626 ¶meters_.config.rtp.rtx.ssrcs); 1627 parameters_.config.rtp.c_name = sp.cname; 1628 parameters_.config.rtp.extensions = rtp_extensions; 1629 parameters_.config.rtp.rtcp_mode = send_params.rtcp.reduced_size 1630 ? webrtc::RtcpMode::kReducedSize 1631 : webrtc::RtcpMode::kCompound; 1632 1633 if (codec_settings) { 1634 SetCodec(*codec_settings); 1635 } 1636 } 1637 1638 WebRtcVideoChannel2::WebRtcVideoSendStream::~WebRtcVideoSendStream() { 1639 DisconnectCapturer(); 1640 if (stream_ != NULL) { 1641 call_->DestroyVideoSendStream(stream_); 1642 } 1643 DestroyVideoEncoder(&allocated_encoder_); 1644 } 1645 1646 static void CreateBlackFrame(webrtc::VideoFrame* video_frame, 1647 int width, 1648 int height) { 1649 video_frame->CreateEmptyFrame(width, height, width, (width + 1) / 2, 1650 (width + 1) / 2); 1651 memset(video_frame->buffer(webrtc::kYPlane), 16, 1652 video_frame->allocated_size(webrtc::kYPlane)); 1653 memset(video_frame->buffer(webrtc::kUPlane), 128, 1654 video_frame->allocated_size(webrtc::kUPlane)); 1655 memset(video_frame->buffer(webrtc::kVPlane), 128, 1656 video_frame->allocated_size(webrtc::kVPlane)); 1657 } 1658 1659 void WebRtcVideoChannel2::WebRtcVideoSendStream::InputFrame( 1660 VideoCapturer* capturer, 1661 const VideoFrame* frame) { 1662 TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::InputFrame"); 1663 webrtc::VideoFrame video_frame(frame->GetVideoFrameBuffer(), 0, 0, 1664 frame->GetVideoRotation()); 1665 rtc::CritScope cs(&lock_); 1666 if (stream_ == NULL) { 1667 // Frame input before send codecs are configured, dropping frame. 1668 return; 1669 } 1670 1671 // Not sending, abort early to prevent expensive reconfigurations while 1672 // setting up codecs etc. 1673 if (!sending_) 1674 return; 1675 1676 if (format_.width == 0) { // Dropping frames. 1677 RTC_DCHECK(format_.height == 0); 1678 LOG(LS_VERBOSE) << "VideoFormat 0x0 set, Dropping frame."; 1679 return; 1680 } 1681 if (muted_) { 1682 // Create a black frame to transmit instead. 1683 CreateBlackFrame(&video_frame, 1684 static_cast<int>(frame->GetWidth()), 1685 static_cast<int>(frame->GetHeight())); 1686 } 1687 1688 int64_t frame_delta_ms = frame->GetTimeStamp() / rtc::kNumNanosecsPerMillisec; 1689 // frame->GetTimeStamp() is essentially a delta, align to webrtc time 1690 if (first_frame_timestamp_ms_ == 0) { 1691 first_frame_timestamp_ms_ = rtc::Time() - frame_delta_ms; 1692 } 1693 1694 last_frame_timestamp_ms_ = first_frame_timestamp_ms_ + frame_delta_ms; 1695 video_frame.set_render_time_ms(last_frame_timestamp_ms_); 1696 // Reconfigure codec if necessary. 1697 SetDimensions( 1698 video_frame.width(), video_frame.height(), capturer->IsScreencast()); 1699 1700 stream_->Input()->IncomingCapturedFrame(video_frame); 1701 } 1702 1703 bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetCapturer( 1704 VideoCapturer* capturer) { 1705 TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::SetCapturer"); 1706 if (!DisconnectCapturer() && capturer == NULL) { 1707 return false; 1708 } 1709 1710 { 1711 rtc::CritScope cs(&lock_); 1712 1713 // Reset timestamps to realign new incoming frames to a webrtc timestamp. A 1714 // new capturer may have a different timestamp delta than the previous one. 1715 first_frame_timestamp_ms_ = 0; 1716 1717 if (capturer == NULL) { 1718 if (stream_ != NULL) { 1719 LOG(LS_VERBOSE) << "Disabling capturer, sending black frame."; 1720 webrtc::VideoFrame black_frame; 1721 1722 CreateBlackFrame(&black_frame, last_dimensions_.width, 1723 last_dimensions_.height); 1724 1725 // Force this black frame not to be dropped due to timestamp order 1726 // check. As IncomingCapturedFrame will drop the frame if this frame's 1727 // timestamp is less than or equal to last frame's timestamp, it is 1728 // necessary to give this black frame a larger timestamp than the 1729 // previous one. 1730 last_frame_timestamp_ms_ += 1731 format_.interval / rtc::kNumNanosecsPerMillisec; 1732 black_frame.set_render_time_ms(last_frame_timestamp_ms_); 1733 stream_->Input()->IncomingCapturedFrame(black_frame); 1734 } 1735 1736 capturer_ = NULL; 1737 return true; 1738 } 1739 1740 capturer_ = capturer; 1741 } 1742 // Lock cannot be held while connecting the capturer to prevent lock-order 1743 // violations. 1744 capturer->SignalVideoFrame.connect(this, &WebRtcVideoSendStream::InputFrame); 1745 return true; 1746 } 1747 1748 bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetVideoFormat( 1749 const VideoFormat& format) { 1750 if ((format.width == 0 || format.height == 0) && 1751 format.width != format.height) { 1752 LOG(LS_ERROR) << "Can't set VideoFormat, width or height is zero (but not " 1753 "both, 0x0 drops frames)."; 1754 return false; 1755 } 1756 1757 rtc::CritScope cs(&lock_); 1758 if (format.width == 0 && format.height == 0) { 1759 LOG(LS_INFO) 1760 << "0x0 resolution selected. Captured frames will be dropped for ssrc: " 1761 << parameters_.config.rtp.ssrcs[0] << "."; 1762 } else { 1763 // TODO(pbos): Fix me, this only affects the last stream! 1764 parameters_.encoder_config.streams.back().max_framerate = 1765 VideoFormat::IntervalToFps(format.interval); 1766 SetDimensions(format.width, format.height, false); 1767 } 1768 1769 format_ = format; 1770 return true; 1771 } 1772 1773 void WebRtcVideoChannel2::WebRtcVideoSendStream::MuteStream(bool mute) { 1774 rtc::CritScope cs(&lock_); 1775 muted_ = mute; 1776 } 1777 1778 bool WebRtcVideoChannel2::WebRtcVideoSendStream::DisconnectCapturer() { 1779 cricket::VideoCapturer* capturer; 1780 { 1781 rtc::CritScope cs(&lock_); 1782 if (capturer_ == NULL) 1783 return false; 1784 1785 if (capturer_->video_adapter() != nullptr) 1786 old_adapt_changes_ += capturer_->video_adapter()->adaptation_changes(); 1787 1788 capturer = capturer_; 1789 capturer_ = NULL; 1790 } 1791 capturer->SignalVideoFrame.disconnect(this); 1792 return true; 1793 } 1794 1795 const std::vector<uint32_t>& 1796 WebRtcVideoChannel2::WebRtcVideoSendStream::GetSsrcs() const { 1797 return ssrcs_; 1798 } 1799 1800 void WebRtcVideoChannel2::WebRtcVideoSendStream::SetApplyRotation( 1801 bool apply_rotation) { 1802 rtc::CritScope cs(&lock_); 1803 if (capturer_ == NULL) 1804 return; 1805 1806 capturer_->SetApplyRotation(apply_rotation); 1807 } 1808 1809 void WebRtcVideoChannel2::WebRtcVideoSendStream::SetOptions( 1810 const VideoOptions& options) { 1811 rtc::CritScope cs(&lock_); 1812 if (parameters_.codec_settings) { 1813 LOG(LS_INFO) << "SetCodecAndOptions because of SetOptions; options=" 1814 << options.ToString(); 1815 SetCodecAndOptions(*parameters_.codec_settings, options); 1816 } else { 1817 parameters_.options = options; 1818 } 1819 } 1820 1821 void WebRtcVideoChannel2::WebRtcVideoSendStream::SetCodec( 1822 const VideoCodecSettings& codec_settings) { 1823 rtc::CritScope cs(&lock_); 1824 LOG(LS_INFO) << "SetCodecAndOptions because of SetCodec."; 1825 SetCodecAndOptions(codec_settings, parameters_.options); 1826 } 1827 1828 webrtc::VideoCodecType CodecTypeFromName(const std::string& name) { 1829 if (CodecNamesEq(name, kVp8CodecName)) { 1830 return webrtc::kVideoCodecVP8; 1831 } else if (CodecNamesEq(name, kVp9CodecName)) { 1832 return webrtc::kVideoCodecVP9; 1833 } else if (CodecNamesEq(name, kH264CodecName)) { 1834 return webrtc::kVideoCodecH264; 1835 } 1836 return webrtc::kVideoCodecUnknown; 1837 } 1838 1839 WebRtcVideoChannel2::WebRtcVideoSendStream::AllocatedEncoder 1840 WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoder( 1841 const VideoCodec& codec) { 1842 webrtc::VideoCodecType type = CodecTypeFromName(codec.name); 1843 1844 // Do not re-create encoders of the same type. 1845 if (type == allocated_encoder_.type && allocated_encoder_.encoder != NULL) { 1846 return allocated_encoder_; 1847 } 1848 1849 if (external_encoder_factory_ != NULL) { 1850 webrtc::VideoEncoder* encoder = 1851 external_encoder_factory_->CreateVideoEncoder(type); 1852 if (encoder != NULL) { 1853 return AllocatedEncoder(encoder, type, true); 1854 } 1855 } 1856 1857 if (type == webrtc::kVideoCodecVP8) { 1858 return AllocatedEncoder( 1859 webrtc::VideoEncoder::Create(webrtc::VideoEncoder::kVp8), type, false); 1860 } else if (type == webrtc::kVideoCodecVP9) { 1861 return AllocatedEncoder( 1862 webrtc::VideoEncoder::Create(webrtc::VideoEncoder::kVp9), type, false); 1863 } else if (type == webrtc::kVideoCodecH264) { 1864 return AllocatedEncoder( 1865 webrtc::VideoEncoder::Create(webrtc::VideoEncoder::kH264), type, false); 1866 } 1867 1868 // This shouldn't happen, we should not be trying to create something we don't 1869 // support. 1870 RTC_DCHECK(false); 1871 return AllocatedEncoder(NULL, webrtc::kVideoCodecUnknown, false); 1872 } 1873 1874 void WebRtcVideoChannel2::WebRtcVideoSendStream::DestroyVideoEncoder( 1875 AllocatedEncoder* encoder) { 1876 if (encoder->external) { 1877 external_encoder_factory_->DestroyVideoEncoder(encoder->external_encoder); 1878 } 1879 delete encoder->encoder; 1880 } 1881 1882 void WebRtcVideoChannel2::WebRtcVideoSendStream::SetCodecAndOptions( 1883 const VideoCodecSettings& codec_settings, 1884 const VideoOptions& options) { 1885 parameters_.encoder_config = 1886 CreateVideoEncoderConfig(last_dimensions_, codec_settings.codec); 1887 if (parameters_.encoder_config.streams.empty()) 1888 return; 1889 1890 format_ = VideoFormat(codec_settings.codec.width, 1891 codec_settings.codec.height, 1892 VideoFormat::FpsToInterval(30), 1893 FOURCC_I420); 1894 1895 AllocatedEncoder new_encoder = CreateVideoEncoder(codec_settings.codec); 1896 parameters_.config.encoder_settings.encoder = new_encoder.encoder; 1897 parameters_.config.encoder_settings.payload_name = codec_settings.codec.name; 1898 parameters_.config.encoder_settings.payload_type = codec_settings.codec.id; 1899 if (new_encoder.external) { 1900 webrtc::VideoCodecType type = CodecTypeFromName(codec_settings.codec.name); 1901 parameters_.config.encoder_settings.internal_source = 1902 external_encoder_factory_->EncoderTypeHasInternalSource(type); 1903 } 1904 parameters_.config.rtp.fec = codec_settings.fec; 1905 1906 // Set RTX payload type if RTX is enabled. 1907 if (!parameters_.config.rtp.rtx.ssrcs.empty()) { 1908 if (codec_settings.rtx_payload_type == -1) { 1909 LOG(LS_WARNING) << "RTX SSRCs configured but there's no configured RTX " 1910 "payload type. Ignoring."; 1911 parameters_.config.rtp.rtx.ssrcs.clear(); 1912 } else { 1913 parameters_.config.rtp.rtx.payload_type = codec_settings.rtx_payload_type; 1914 } 1915 } 1916 1917 parameters_.config.rtp.nack.rtp_history_ms = 1918 HasNack(codec_settings.codec) ? kNackHistoryMs : 0; 1919 1920 RTC_CHECK(options.suspend_below_min_bitrate); 1921 parameters_.config.suspend_below_min_bitrate = 1922 *options.suspend_below_min_bitrate; 1923 1924 parameters_.codec_settings = 1925 rtc::Optional<WebRtcVideoChannel2::VideoCodecSettings>(codec_settings); 1926 parameters_.options = options; 1927 1928 LOG(LS_INFO) 1929 << "RecreateWebRtcStream (send) because of SetCodecAndOptions; options=" 1930 << options.ToString(); 1931 RecreateWebRtcStream(); 1932 if (allocated_encoder_.encoder != new_encoder.encoder) { 1933 DestroyVideoEncoder(&allocated_encoder_); 1934 allocated_encoder_ = new_encoder; 1935 } 1936 } 1937 1938 void WebRtcVideoChannel2::WebRtcVideoSendStream::SetRtpExtensions( 1939 const std::vector<webrtc::RtpExtension>& rtp_extensions) { 1940 rtc::CritScope cs(&lock_); 1941 parameters_.config.rtp.extensions = rtp_extensions; 1942 if (stream_ != nullptr) { 1943 LOG(LS_INFO) << "RecreateWebRtcStream (send) because of SetRtpExtensions"; 1944 RecreateWebRtcStream(); 1945 } 1946 } 1947 1948 void WebRtcVideoChannel2::WebRtcVideoSendStream::SetSendParameters( 1949 const VideoSendParameters& send_params) { 1950 rtc::CritScope cs(&lock_); 1951 parameters_.config.rtp.rtcp_mode = send_params.rtcp.reduced_size 1952 ? webrtc::RtcpMode::kReducedSize 1953 : webrtc::RtcpMode::kCompound; 1954 if (stream_ != nullptr) { 1955 LOG(LS_INFO) << "RecreateWebRtcStream (send) because of SetSendParameters"; 1956 RecreateWebRtcStream(); 1957 } 1958 } 1959 1960 webrtc::VideoEncoderConfig 1961 WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoderConfig( 1962 const Dimensions& dimensions, 1963 const VideoCodec& codec) const { 1964 webrtc::VideoEncoderConfig encoder_config; 1965 if (dimensions.is_screencast) { 1966 RTC_CHECK(parameters_.options.screencast_min_bitrate); 1967 encoder_config.min_transmit_bitrate_bps = 1968 *parameters_.options.screencast_min_bitrate * 1000; 1969 encoder_config.content_type = 1970 webrtc::VideoEncoderConfig::ContentType::kScreen; 1971 } else { 1972 encoder_config.min_transmit_bitrate_bps = 0; 1973 encoder_config.content_type = 1974 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo; 1975 } 1976 1977 // Restrict dimensions according to codec max. 1978 int width = dimensions.width; 1979 int height = dimensions.height; 1980 if (!dimensions.is_screencast) { 1981 if (codec.width < width) 1982 width = codec.width; 1983 if (codec.height < height) 1984 height = codec.height; 1985 } 1986 1987 VideoCodec clamped_codec = codec; 1988 clamped_codec.width = width; 1989 clamped_codec.height = height; 1990 1991 // By default, the stream count for the codec configuration should match the 1992 // number of negotiated ssrcs. But if the codec is blacklisted for simulcast 1993 // or a screencast, only configure a single stream. 1994 size_t stream_count = parameters_.config.rtp.ssrcs.size(); 1995 if (IsCodecBlacklistedForSimulcast(codec.name) || dimensions.is_screencast) { 1996 stream_count = 1; 1997 } 1998 1999 encoder_config.streams = 2000 CreateVideoStreams(clamped_codec, parameters_.options, 2001 parameters_.max_bitrate_bps, stream_count); 2002 2003 // Conference mode screencast uses 2 temporal layers split at 100kbit. 2004 if (parameters_.options.conference_mode.value_or(false) && 2005 dimensions.is_screencast && encoder_config.streams.size() == 1) { 2006 ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault(); 2007 2008 // For screenshare in conference mode, tl0 and tl1 bitrates are piggybacked 2009 // on the VideoCodec struct as target and max bitrates, respectively. 2010 // See eg. webrtc::VP8EncoderImpl::SetRates(). 2011 encoder_config.streams[0].target_bitrate_bps = 2012 config.tl0_bitrate_kbps * 1000; 2013 encoder_config.streams[0].max_bitrate_bps = config.tl1_bitrate_kbps * 1000; 2014 encoder_config.streams[0].temporal_layer_thresholds_bps.clear(); 2015 encoder_config.streams[0].temporal_layer_thresholds_bps.push_back( 2016 config.tl0_bitrate_kbps * 1000); 2017 } 2018 return encoder_config; 2019 } 2020 2021 void WebRtcVideoChannel2::WebRtcVideoSendStream::SetDimensions( 2022 int width, 2023 int height, 2024 bool is_screencast) { 2025 if (last_dimensions_.width == width && last_dimensions_.height == height && 2026 last_dimensions_.is_screencast == is_screencast) { 2027 // Configured using the same parameters, do not reconfigure. 2028 return; 2029 } 2030 LOG(LS_INFO) << "SetDimensions: " << width << "x" << height 2031 << (is_screencast ? " (screencast)" : " (not screencast)"); 2032 2033 last_dimensions_.width = width; 2034 last_dimensions_.height = height; 2035 last_dimensions_.is_screencast = is_screencast; 2036 2037 RTC_DCHECK(!parameters_.encoder_config.streams.empty()); 2038 2039 RTC_CHECK(parameters_.codec_settings); 2040 VideoCodecSettings codec_settings = *parameters_.codec_settings; 2041 2042 webrtc::VideoEncoderConfig encoder_config = 2043 CreateVideoEncoderConfig(last_dimensions_, codec_settings.codec); 2044 2045 encoder_config.encoder_specific_settings = ConfigureVideoEncoderSettings( 2046 codec_settings.codec, parameters_.options, is_screencast); 2047 2048 bool stream_reconfigured = stream_->ReconfigureVideoEncoder(encoder_config); 2049 2050 encoder_config.encoder_specific_settings = NULL; 2051 2052 if (!stream_reconfigured) { 2053 LOG(LS_WARNING) << "Failed to reconfigure video encoder for dimensions: " 2054 << width << "x" << height; 2055 return; 2056 } 2057 2058 parameters_.encoder_config = encoder_config; 2059 } 2060 2061 void WebRtcVideoChannel2::WebRtcVideoSendStream::Start() { 2062 rtc::CritScope cs(&lock_); 2063 RTC_DCHECK(stream_ != NULL); 2064 stream_->Start(); 2065 sending_ = true; 2066 } 2067 2068 void WebRtcVideoChannel2::WebRtcVideoSendStream::Stop() { 2069 rtc::CritScope cs(&lock_); 2070 if (stream_ != NULL) { 2071 stream_->Stop(); 2072 } 2073 sending_ = false; 2074 } 2075 2076 VideoSenderInfo 2077 WebRtcVideoChannel2::WebRtcVideoSendStream::GetVideoSenderInfo() { 2078 VideoSenderInfo info; 2079 webrtc::VideoSendStream::Stats stats; 2080 { 2081 rtc::CritScope cs(&lock_); 2082 for (uint32_t ssrc : parameters_.config.rtp.ssrcs) 2083 info.add_ssrc(ssrc); 2084 2085 if (parameters_.codec_settings) 2086 info.codec_name = parameters_.codec_settings->codec.name; 2087 for (size_t i = 0; i < parameters_.encoder_config.streams.size(); ++i) { 2088 if (i == parameters_.encoder_config.streams.size() - 1) { 2089 info.preferred_bitrate += 2090 parameters_.encoder_config.streams[i].max_bitrate_bps; 2091 } else { 2092 info.preferred_bitrate += 2093 parameters_.encoder_config.streams[i].target_bitrate_bps; 2094 } 2095 } 2096 2097 if (stream_ == NULL) 2098 return info; 2099 2100 stats = stream_->GetStats(); 2101 2102 info.adapt_changes = old_adapt_changes_; 2103 info.adapt_reason = CoordinatedVideoAdapter::ADAPTREASON_NONE; 2104 2105 if (capturer_ != NULL) { 2106 if (!capturer_->IsMuted()) { 2107 VideoFormat last_captured_frame_format; 2108 capturer_->GetStats(&info.adapt_frame_drops, &info.effects_frame_drops, 2109 &info.capturer_frame_time, 2110 &last_captured_frame_format); 2111 info.input_frame_width = last_captured_frame_format.width; 2112 info.input_frame_height = last_captured_frame_format.height; 2113 } 2114 if (capturer_->video_adapter() != nullptr) { 2115 info.adapt_changes += capturer_->video_adapter()->adaptation_changes(); 2116 info.adapt_reason = capturer_->video_adapter()->adapt_reason(); 2117 } 2118 } 2119 } 2120 2121 // Get bandwidth limitation info from stream_->GetStats(). 2122 // Input resolution (output from video_adapter) can be further scaled down or 2123 // higher video layer(s) can be dropped due to bitrate constraints. 2124 // Note, adapt_changes only include changes from the video_adapter. 2125 if (stats.bw_limited_resolution) 2126 info.adapt_reason |= CoordinatedVideoAdapter::ADAPTREASON_BANDWIDTH; 2127 2128 info.encoder_implementation_name = stats.encoder_implementation_name; 2129 info.ssrc_groups = ssrc_groups_; 2130 info.framerate_input = stats.input_frame_rate; 2131 info.framerate_sent = stats.encode_frame_rate; 2132 info.avg_encode_ms = stats.avg_encode_time_ms; 2133 info.encode_usage_percent = stats.encode_usage_percent; 2134 2135 info.nominal_bitrate = stats.media_bitrate_bps; 2136 2137 info.send_frame_width = 0; 2138 info.send_frame_height = 0; 2139 for (std::map<uint32_t, webrtc::VideoSendStream::StreamStats>::iterator it = 2140 stats.substreams.begin(); 2141 it != stats.substreams.end(); ++it) { 2142 // TODO(pbos): Wire up additional stats, such as padding bytes. 2143 webrtc::VideoSendStream::StreamStats stream_stats = it->second; 2144 info.bytes_sent += stream_stats.rtp_stats.transmitted.payload_bytes + 2145 stream_stats.rtp_stats.transmitted.header_bytes + 2146 stream_stats.rtp_stats.transmitted.padding_bytes; 2147 info.packets_sent += stream_stats.rtp_stats.transmitted.packets; 2148 info.packets_lost += stream_stats.rtcp_stats.cumulative_lost; 2149 if (stream_stats.width > info.send_frame_width) 2150 info.send_frame_width = stream_stats.width; 2151 if (stream_stats.height > info.send_frame_height) 2152 info.send_frame_height = stream_stats.height; 2153 info.firs_rcvd += stream_stats.rtcp_packet_type_counts.fir_packets; 2154 info.nacks_rcvd += stream_stats.rtcp_packet_type_counts.nack_packets; 2155 info.plis_rcvd += stream_stats.rtcp_packet_type_counts.pli_packets; 2156 } 2157 2158 if (!stats.substreams.empty()) { 2159 // TODO(pbos): Report fraction lost per SSRC. 2160 webrtc::VideoSendStream::StreamStats first_stream_stats = 2161 stats.substreams.begin()->second; 2162 info.fraction_lost = 2163 static_cast<float>(first_stream_stats.rtcp_stats.fraction_lost) / 2164 (1 << 8); 2165 } 2166 2167 return info; 2168 } 2169 2170 void WebRtcVideoChannel2::WebRtcVideoSendStream::FillBandwidthEstimationInfo( 2171 BandwidthEstimationInfo* bwe_info) { 2172 rtc::CritScope cs(&lock_); 2173 if (stream_ == NULL) { 2174 return; 2175 } 2176 webrtc::VideoSendStream::Stats stats = stream_->GetStats(); 2177 for (std::map<uint32_t, webrtc::VideoSendStream::StreamStats>::iterator it = 2178 stats.substreams.begin(); 2179 it != stats.substreams.end(); ++it) { 2180 bwe_info->transmit_bitrate += it->second.total_bitrate_bps; 2181 bwe_info->retransmit_bitrate += it->second.retransmit_bitrate_bps; 2182 } 2183 bwe_info->target_enc_bitrate += stats.target_media_bitrate_bps; 2184 bwe_info->actual_enc_bitrate += stats.media_bitrate_bps; 2185 } 2186 2187 void WebRtcVideoChannel2::WebRtcVideoSendStream::SetMaxBitrateBps( 2188 int max_bitrate_bps) { 2189 rtc::CritScope cs(&lock_); 2190 parameters_.max_bitrate_bps = max_bitrate_bps; 2191 2192 // No need to reconfigure if the stream hasn't been configured yet. 2193 if (parameters_.encoder_config.streams.empty()) 2194 return; 2195 2196 // Force a stream reconfigure to set the new max bitrate. 2197 int width = last_dimensions_.width; 2198 last_dimensions_.width = 0; 2199 SetDimensions(width, last_dimensions_.height, last_dimensions_.is_screencast); 2200 } 2201 2202 void WebRtcVideoChannel2::WebRtcVideoSendStream::RecreateWebRtcStream() { 2203 if (stream_ != NULL) { 2204 call_->DestroyVideoSendStream(stream_); 2205 } 2206 2207 RTC_CHECK(parameters_.codec_settings); 2208 parameters_.encoder_config.encoder_specific_settings = 2209 ConfigureVideoEncoderSettings( 2210 parameters_.codec_settings->codec, parameters_.options, 2211 parameters_.encoder_config.content_type == 2212 webrtc::VideoEncoderConfig::ContentType::kScreen); 2213 2214 webrtc::VideoSendStream::Config config = parameters_.config; 2215 if (!config.rtp.rtx.ssrcs.empty() && config.rtp.rtx.payload_type == -1) { 2216 LOG(LS_WARNING) << "RTX SSRCs configured but there's no configured RTX " 2217 "payload type the set codec. Ignoring RTX."; 2218 config.rtp.rtx.ssrcs.clear(); 2219 } 2220 stream_ = call_->CreateVideoSendStream(config, parameters_.encoder_config); 2221 2222 parameters_.encoder_config.encoder_specific_settings = NULL; 2223 2224 if (sending_) { 2225 stream_->Start(); 2226 } 2227 } 2228 2229 WebRtcVideoChannel2::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( 2230 webrtc::Call* call, 2231 const StreamParams& sp, 2232 const webrtc::VideoReceiveStream::Config& config, 2233 WebRtcVideoDecoderFactory* external_decoder_factory, 2234 bool default_stream, 2235 const std::vector<VideoCodecSettings>& recv_codecs, 2236 bool disable_prerenderer_smoothing) 2237 : call_(call), 2238 ssrcs_(sp.ssrcs), 2239 ssrc_groups_(sp.ssrc_groups), 2240 stream_(NULL), 2241 default_stream_(default_stream), 2242 config_(config), 2243 external_decoder_factory_(external_decoder_factory), 2244 disable_prerenderer_smoothing_(disable_prerenderer_smoothing), 2245 renderer_(NULL), 2246 last_width_(-1), 2247 last_height_(-1), 2248 first_frame_timestamp_(-1), 2249 estimated_remote_start_ntp_time_ms_(0) { 2250 config_.renderer = this; 2251 // SetRecvCodecs will also reset (start) the VideoReceiveStream. 2252 LOG(LS_INFO) << "SetRecvCodecs (recv) because we are creating the receive " 2253 "stream for the first time: " 2254 << CodecSettingsVectorToString(recv_codecs); 2255 SetRecvCodecs(recv_codecs); 2256 } 2257 2258 WebRtcVideoChannel2::WebRtcVideoReceiveStream::AllocatedDecoder:: 2259 AllocatedDecoder(webrtc::VideoDecoder* decoder, 2260 webrtc::VideoCodecType type, 2261 bool external) 2262 : decoder(decoder), 2263 external_decoder(nullptr), 2264 type(type), 2265 external(external) { 2266 if (external) { 2267 external_decoder = decoder; 2268 this->decoder = 2269 new webrtc::VideoDecoderSoftwareFallbackWrapper(type, external_decoder); 2270 } 2271 } 2272 2273 WebRtcVideoChannel2::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() { 2274 call_->DestroyVideoReceiveStream(stream_); 2275 ClearDecoders(&allocated_decoders_); 2276 } 2277 2278 const std::vector<uint32_t>& 2279 WebRtcVideoChannel2::WebRtcVideoReceiveStream::GetSsrcs() const { 2280 return ssrcs_; 2281 } 2282 2283 WebRtcVideoChannel2::WebRtcVideoReceiveStream::AllocatedDecoder 2284 WebRtcVideoChannel2::WebRtcVideoReceiveStream::CreateOrReuseVideoDecoder( 2285 std::vector<AllocatedDecoder>* old_decoders, 2286 const VideoCodec& codec) { 2287 webrtc::VideoCodecType type = CodecTypeFromName(codec.name); 2288 2289 for (size_t i = 0; i < old_decoders->size(); ++i) { 2290 if ((*old_decoders)[i].type == type) { 2291 AllocatedDecoder decoder = (*old_decoders)[i]; 2292 (*old_decoders)[i] = old_decoders->back(); 2293 old_decoders->pop_back(); 2294 return decoder; 2295 } 2296 } 2297 2298 if (external_decoder_factory_ != NULL) { 2299 webrtc::VideoDecoder* decoder = 2300 external_decoder_factory_->CreateVideoDecoder(type); 2301 if (decoder != NULL) { 2302 return AllocatedDecoder(decoder, type, true); 2303 } 2304 } 2305 2306 if (type == webrtc::kVideoCodecVP8) { 2307 return AllocatedDecoder( 2308 webrtc::VideoDecoder::Create(webrtc::VideoDecoder::kVp8), type, false); 2309 } 2310 2311 if (type == webrtc::kVideoCodecVP9) { 2312 return AllocatedDecoder( 2313 webrtc::VideoDecoder::Create(webrtc::VideoDecoder::kVp9), type, false); 2314 } 2315 2316 if (type == webrtc::kVideoCodecH264) { 2317 return AllocatedDecoder( 2318 webrtc::VideoDecoder::Create(webrtc::VideoDecoder::kH264), type, false); 2319 } 2320 2321 // This shouldn't happen, we should not be trying to create something we don't 2322 // support. 2323 RTC_DCHECK(false); 2324 return AllocatedDecoder(NULL, webrtc::kVideoCodecUnknown, false); 2325 } 2326 2327 void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRecvCodecs( 2328 const std::vector<VideoCodecSettings>& recv_codecs) { 2329 std::vector<AllocatedDecoder> old_decoders = allocated_decoders_; 2330 allocated_decoders_.clear(); 2331 config_.decoders.clear(); 2332 for (size_t i = 0; i < recv_codecs.size(); ++i) { 2333 AllocatedDecoder allocated_decoder = 2334 CreateOrReuseVideoDecoder(&old_decoders, recv_codecs[i].codec); 2335 allocated_decoders_.push_back(allocated_decoder); 2336 2337 webrtc::VideoReceiveStream::Decoder decoder; 2338 decoder.decoder = allocated_decoder.decoder; 2339 decoder.payload_type = recv_codecs[i].codec.id; 2340 decoder.payload_name = recv_codecs[i].codec.name; 2341 config_.decoders.push_back(decoder); 2342 } 2343 2344 // TODO(pbos): Reconfigure RTX based on incoming recv_codecs. 2345 config_.rtp.fec = recv_codecs.front().fec; 2346 config_.rtp.nack.rtp_history_ms = 2347 HasNack(recv_codecs.begin()->codec) ? kNackHistoryMs : 0; 2348 2349 LOG(LS_INFO) << "RecreateWebRtcStream (recv) because of SetRecvCodecs: " 2350 << CodecSettingsVectorToString(recv_codecs); 2351 RecreateWebRtcStream(); 2352 ClearDecoders(&old_decoders); 2353 } 2354 2355 void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetLocalSsrc( 2356 uint32_t local_ssrc) { 2357 // TODO(pbos): Consider turning this sanity check into a RTC_DCHECK. You 2358 // should not be able to create a sender with the same SSRC as a receiver, but 2359 // right now this can't be done due to unittests depending on receiving what 2360 // they are sending from the same MediaChannel. 2361 if (local_ssrc == config_.rtp.remote_ssrc) { 2362 LOG(LS_INFO) << "Ignoring call to SetLocalSsrc because parameters are " 2363 "unchanged; local_ssrc=" << local_ssrc; 2364 return; 2365 } 2366 2367 config_.rtp.local_ssrc = local_ssrc; 2368 LOG(LS_INFO) 2369 << "RecreateWebRtcStream (recv) because of SetLocalSsrc; local_ssrc=" 2370 << local_ssrc; 2371 RecreateWebRtcStream(); 2372 } 2373 2374 void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetFeedbackParameters( 2375 bool nack_enabled, 2376 bool remb_enabled, 2377 bool transport_cc_enabled) { 2378 int nack_history_ms = nack_enabled ? kNackHistoryMs : 0; 2379 if (config_.rtp.nack.rtp_history_ms == nack_history_ms && 2380 config_.rtp.remb == remb_enabled && 2381 config_.rtp.transport_cc == transport_cc_enabled) { 2382 LOG(LS_INFO) 2383 << "Ignoring call to SetFeedbackParameters because parameters are " 2384 "unchanged; nack=" 2385 << nack_enabled << ", remb=" << remb_enabled 2386 << ", transport_cc=" << transport_cc_enabled; 2387 return; 2388 } 2389 config_.rtp.remb = remb_enabled; 2390 config_.rtp.nack.rtp_history_ms = nack_history_ms; 2391 config_.rtp.transport_cc = transport_cc_enabled; 2392 LOG(LS_INFO) 2393 << "RecreateWebRtcStream (recv) because of SetFeedbackParameters; nack=" 2394 << nack_enabled << ", remb=" << remb_enabled 2395 << ", transport_cc=" << transport_cc_enabled; 2396 RecreateWebRtcStream(); 2397 } 2398 2399 void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRtpExtensions( 2400 const std::vector<webrtc::RtpExtension>& extensions) { 2401 config_.rtp.extensions = extensions; 2402 LOG(LS_INFO) << "RecreateWebRtcStream (recv) because of SetRtpExtensions"; 2403 RecreateWebRtcStream(); 2404 } 2405 2406 void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRecvParameters( 2407 const VideoRecvParameters& recv_params) { 2408 config_.rtp.rtcp_mode = recv_params.rtcp.reduced_size 2409 ? webrtc::RtcpMode::kReducedSize 2410 : webrtc::RtcpMode::kCompound; 2411 LOG(LS_INFO) << "RecreateWebRtcStream (recv) because of SetRecvParameters"; 2412 RecreateWebRtcStream(); 2413 } 2414 2415 void WebRtcVideoChannel2::WebRtcVideoReceiveStream::RecreateWebRtcStream() { 2416 if (stream_ != NULL) { 2417 call_->DestroyVideoReceiveStream(stream_); 2418 } 2419 stream_ = call_->CreateVideoReceiveStream(config_); 2420 stream_->Start(); 2421 } 2422 2423 void WebRtcVideoChannel2::WebRtcVideoReceiveStream::ClearDecoders( 2424 std::vector<AllocatedDecoder>* allocated_decoders) { 2425 for (size_t i = 0; i < allocated_decoders->size(); ++i) { 2426 if ((*allocated_decoders)[i].external) { 2427 external_decoder_factory_->DestroyVideoDecoder( 2428 (*allocated_decoders)[i].external_decoder); 2429 } 2430 delete (*allocated_decoders)[i].decoder; 2431 } 2432 allocated_decoders->clear(); 2433 } 2434 2435 void WebRtcVideoChannel2::WebRtcVideoReceiveStream::RenderFrame( 2436 const webrtc::VideoFrame& frame, 2437 int time_to_render_ms) { 2438 rtc::CritScope crit(&renderer_lock_); 2439 2440 if (first_frame_timestamp_ < 0) 2441 first_frame_timestamp_ = frame.timestamp(); 2442 int64_t rtp_time_elapsed_since_first_frame = 2443 (timestamp_wraparound_handler_.Unwrap(frame.timestamp()) - 2444 first_frame_timestamp_); 2445 int64_t elapsed_time_ms = rtp_time_elapsed_since_first_frame / 2446 (cricket::kVideoCodecClockrate / 1000); 2447 if (frame.ntp_time_ms() > 0) 2448 estimated_remote_start_ntp_time_ms_ = frame.ntp_time_ms() - elapsed_time_ms; 2449 2450 if (renderer_ == NULL) { 2451 LOG(LS_WARNING) << "VideoReceiveStream not connected to a VideoRenderer."; 2452 return; 2453 } 2454 2455 if (frame.width() != last_width_ || frame.height() != last_height_) { 2456 SetSize(frame.width(), frame.height()); 2457 } 2458 2459 const WebRtcVideoFrame render_frame( 2460 frame.video_frame_buffer(), 2461 frame.render_time_ms() * rtc::kNumNanosecsPerMillisec, frame.rotation()); 2462 renderer_->RenderFrame(&render_frame); 2463 } 2464 2465 bool WebRtcVideoChannel2::WebRtcVideoReceiveStream::IsTextureSupported() const { 2466 return true; 2467 } 2468 2469 bool WebRtcVideoChannel2::WebRtcVideoReceiveStream::SmoothsRenderedFrames() 2470 const { 2471 return disable_prerenderer_smoothing_; 2472 } 2473 2474 bool WebRtcVideoChannel2::WebRtcVideoReceiveStream::IsDefaultStream() const { 2475 return default_stream_; 2476 } 2477 2478 void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRenderer( 2479 cricket::VideoRenderer* renderer) { 2480 rtc::CritScope crit(&renderer_lock_); 2481 renderer_ = renderer; 2482 if (renderer_ != NULL && last_width_ != -1) { 2483 SetSize(last_width_, last_height_); 2484 } 2485 } 2486 2487 VideoRenderer* WebRtcVideoChannel2::WebRtcVideoReceiveStream::GetRenderer() { 2488 // TODO(pbos): Remove GetRenderer and all uses of it, it's thread-unsafe by 2489 // design. 2490 rtc::CritScope crit(&renderer_lock_); 2491 return renderer_; 2492 } 2493 2494 void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetSize(int width, 2495 int height) { 2496 rtc::CritScope crit(&renderer_lock_); 2497 if (!renderer_->SetSize(width, height, 0)) { 2498 LOG(LS_ERROR) << "Could not set renderer size."; 2499 } 2500 last_width_ = width; 2501 last_height_ = height; 2502 } 2503 2504 std::string 2505 WebRtcVideoChannel2::WebRtcVideoReceiveStream::GetCodecNameFromPayloadType( 2506 int payload_type) { 2507 for (const webrtc::VideoReceiveStream::Decoder& decoder : config_.decoders) { 2508 if (decoder.payload_type == payload_type) { 2509 return decoder.payload_name; 2510 } 2511 } 2512 return ""; 2513 } 2514 2515 VideoReceiverInfo 2516 WebRtcVideoChannel2::WebRtcVideoReceiveStream::GetVideoReceiverInfo() { 2517 VideoReceiverInfo info; 2518 info.ssrc_groups = ssrc_groups_; 2519 info.add_ssrc(config_.rtp.remote_ssrc); 2520 webrtc::VideoReceiveStream::Stats stats = stream_->GetStats(); 2521 info.decoder_implementation_name = stats.decoder_implementation_name; 2522 info.bytes_rcvd = stats.rtp_stats.transmitted.payload_bytes + 2523 stats.rtp_stats.transmitted.header_bytes + 2524 stats.rtp_stats.transmitted.padding_bytes; 2525 info.packets_rcvd = stats.rtp_stats.transmitted.packets; 2526 info.packets_lost = stats.rtcp_stats.cumulative_lost; 2527 info.fraction_lost = 2528 static_cast<float>(stats.rtcp_stats.fraction_lost) / (1 << 8); 2529 2530 info.framerate_rcvd = stats.network_frame_rate; 2531 info.framerate_decoded = stats.decode_frame_rate; 2532 info.framerate_output = stats.render_frame_rate; 2533 2534 { 2535 rtc::CritScope frame_cs(&renderer_lock_); 2536 info.frame_width = last_width_; 2537 info.frame_height = last_height_; 2538 info.capture_start_ntp_time_ms = estimated_remote_start_ntp_time_ms_; 2539 } 2540 2541 info.decode_ms = stats.decode_ms; 2542 info.max_decode_ms = stats.max_decode_ms; 2543 info.current_delay_ms = stats.current_delay_ms; 2544 info.target_delay_ms = stats.target_delay_ms; 2545 info.jitter_buffer_ms = stats.jitter_buffer_ms; 2546 info.min_playout_delay_ms = stats.min_playout_delay_ms; 2547 info.render_delay_ms = stats.render_delay_ms; 2548 2549 info.codec_name = GetCodecNameFromPayloadType(stats.current_payload_type); 2550 2551 info.firs_sent = stats.rtcp_packet_type_counts.fir_packets; 2552 info.plis_sent = stats.rtcp_packet_type_counts.pli_packets; 2553 info.nacks_sent = stats.rtcp_packet_type_counts.nack_packets; 2554 2555 return info; 2556 } 2557 2558 WebRtcVideoChannel2::VideoCodecSettings::VideoCodecSettings() 2559 : rtx_payload_type(-1) {} 2560 2561 bool WebRtcVideoChannel2::VideoCodecSettings::operator==( 2562 const WebRtcVideoChannel2::VideoCodecSettings& other) const { 2563 return codec == other.codec && 2564 fec.ulpfec_payload_type == other.fec.ulpfec_payload_type && 2565 fec.red_payload_type == other.fec.red_payload_type && 2566 fec.red_rtx_payload_type == other.fec.red_rtx_payload_type && 2567 rtx_payload_type == other.rtx_payload_type; 2568 } 2569 2570 bool WebRtcVideoChannel2::VideoCodecSettings::operator!=( 2571 const WebRtcVideoChannel2::VideoCodecSettings& other) const { 2572 return !(*this == other); 2573 } 2574 2575 std::vector<WebRtcVideoChannel2::VideoCodecSettings> 2576 WebRtcVideoChannel2::MapCodecs(const std::vector<VideoCodec>& codecs) { 2577 RTC_DCHECK(!codecs.empty()); 2578 2579 std::vector<VideoCodecSettings> video_codecs; 2580 std::map<int, bool> payload_used; 2581 std::map<int, VideoCodec::CodecType> payload_codec_type; 2582 // |rtx_mapping| maps video payload type to rtx payload type. 2583 std::map<int, int> rtx_mapping; 2584 2585 webrtc::FecConfig fec_settings; 2586 2587 for (size_t i = 0; i < codecs.size(); ++i) { 2588 const VideoCodec& in_codec = codecs[i]; 2589 int payload_type = in_codec.id; 2590 2591 if (payload_used[payload_type]) { 2592 LOG(LS_ERROR) << "Payload type already registered: " 2593 << in_codec.ToString(); 2594 return std::vector<VideoCodecSettings>(); 2595 } 2596 payload_used[payload_type] = true; 2597 payload_codec_type[payload_type] = in_codec.GetCodecType(); 2598 2599 switch (in_codec.GetCodecType()) { 2600 case VideoCodec::CODEC_RED: { 2601 // RED payload type, should not have duplicates. 2602 RTC_DCHECK(fec_settings.red_payload_type == -1); 2603 fec_settings.red_payload_type = in_codec.id; 2604 continue; 2605 } 2606 2607 case VideoCodec::CODEC_ULPFEC: { 2608 // ULPFEC payload type, should not have duplicates. 2609 RTC_DCHECK(fec_settings.ulpfec_payload_type == -1); 2610 fec_settings.ulpfec_payload_type = in_codec.id; 2611 continue; 2612 } 2613 2614 case VideoCodec::CODEC_RTX: { 2615 int associated_payload_type; 2616 if (!in_codec.GetParam(kCodecParamAssociatedPayloadType, 2617 &associated_payload_type) || 2618 !IsValidRtpPayloadType(associated_payload_type)) { 2619 LOG(LS_ERROR) 2620 << "RTX codec with invalid or no associated payload type: " 2621 << in_codec.ToString(); 2622 return std::vector<VideoCodecSettings>(); 2623 } 2624 rtx_mapping[associated_payload_type] = in_codec.id; 2625 continue; 2626 } 2627 2628 case VideoCodec::CODEC_VIDEO: 2629 break; 2630 } 2631 2632 video_codecs.push_back(VideoCodecSettings()); 2633 video_codecs.back().codec = in_codec; 2634 } 2635 2636 // One of these codecs should have been a video codec. Only having FEC 2637 // parameters into this code is a logic error. 2638 RTC_DCHECK(!video_codecs.empty()); 2639 2640 for (std::map<int, int>::const_iterator it = rtx_mapping.begin(); 2641 it != rtx_mapping.end(); 2642 ++it) { 2643 if (!payload_used[it->first]) { 2644 LOG(LS_ERROR) << "RTX mapped to payload not in codec list."; 2645 return std::vector<VideoCodecSettings>(); 2646 } 2647 if (payload_codec_type[it->first] != VideoCodec::CODEC_VIDEO && 2648 payload_codec_type[it->first] != VideoCodec::CODEC_RED) { 2649 LOG(LS_ERROR) << "RTX not mapped to regular video codec or RED codec."; 2650 return std::vector<VideoCodecSettings>(); 2651 } 2652 2653 if (it->first == fec_settings.red_payload_type) { 2654 fec_settings.red_rtx_payload_type = it->second; 2655 } 2656 } 2657 2658 for (size_t i = 0; i < video_codecs.size(); ++i) { 2659 video_codecs[i].fec = fec_settings; 2660 if (rtx_mapping[video_codecs[i].codec.id] != 0 && 2661 rtx_mapping[video_codecs[i].codec.id] != 2662 fec_settings.red_payload_type) { 2663 video_codecs[i].rtx_payload_type = rtx_mapping[video_codecs[i].codec.id]; 2664 } 2665 } 2666 2667 return video_codecs; 2668 } 2669 2670 } // namespace cricket 2671 2672 #endif // HAVE_WEBRTC_VIDEO 2673