1 /* 2 * libjingle 3 * Copyright 2004 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #ifdef HAVE_WEBRTC_VIDEO 29 #include "talk/media/webrtc/webrtcvideoengine.h" 30 31 #ifdef HAVE_CONFIG_H 32 #include <config.h> 33 #endif 34 35 #include <math.h> 36 #include <set> 37 38 #include "talk/base/basictypes.h" 39 #include "talk/base/buffer.h" 40 #include "talk/base/byteorder.h" 41 #include "talk/base/common.h" 42 #include "talk/base/cpumonitor.h" 43 #include "talk/base/logging.h" 44 #include "talk/base/stringutils.h" 45 #include "talk/base/thread.h" 46 #include "talk/base/timeutils.h" 47 #include "talk/media/base/constants.h" 48 #include "talk/media/base/rtputils.h" 49 #include "talk/media/base/streamparams.h" 50 #include "talk/media/base/videoadapter.h" 51 #include "talk/media/base/videocapturer.h" 52 #include "talk/media/base/videorenderer.h" 53 #include "talk/media/devices/filevideocapturer.h" 54 #include "talk/media/webrtc/webrtcpassthroughrender.h" 55 #include "talk/media/webrtc/webrtctexturevideoframe.h" 56 #include "talk/media/webrtc/webrtcvideocapturer.h" 57 #include "talk/media/webrtc/webrtcvideodecoderfactory.h" 58 #include "talk/media/webrtc/webrtcvideoencoderfactory.h" 59 #include "talk/media/webrtc/webrtcvideoframe.h" 60 #include "talk/media/webrtc/webrtcvie.h" 61 #include "talk/media/webrtc/webrtcvoe.h" 62 #include "talk/media/webrtc/webrtcvoiceengine.h" 63 64 #if !defined(LIBPEERCONNECTION_LIB) 65 #ifndef HAVE_WEBRTC_VIDEO 66 #error Need webrtc video 67 #endif 68 #include "talk/media/webrtc/webrtcmediaengine.h" 69 70 WRME_EXPORT 71 cricket::MediaEngineInterface* CreateWebRtcMediaEngine( 72 webrtc::AudioDeviceModule* adm, webrtc::AudioDeviceModule* adm_sc, 73 cricket::WebRtcVideoEncoderFactory* encoder_factory, 74 cricket::WebRtcVideoDecoderFactory* decoder_factory) { 75 return new cricket::WebRtcMediaEngine(adm, adm_sc, encoder_factory, 76 decoder_factory); 77 } 78 79 WRME_EXPORT 80 void DestroyWebRtcMediaEngine(cricket::MediaEngineInterface* media_engine) { 81 delete static_cast<cricket::WebRtcMediaEngine*>(media_engine); 82 } 83 #endif 84 85 86 namespace cricket { 87 88 89 static const int kDefaultLogSeverity = talk_base::LS_WARNING; 90 91 static const int kMinVideoBitrate = 50; 92 static const int kStartVideoBitrate = 300; 93 static const int kMaxVideoBitrate = 2000; 94 static const int kDefaultConferenceModeMaxVideoBitrate = 500; 95 96 // Controlled by exp, try a super low minimum bitrate for poor connections. 97 static const int kLowerMinBitrate = 30; 98 99 static const int kVideoMtu = 1200; 100 101 static const int kVideoRtpBufferSize = 65536; 102 103 static const char kVp8PayloadName[] = "VP8"; 104 static const char kRedPayloadName[] = "red"; 105 static const char kFecPayloadName[] = "ulpfec"; 106 107 static const int kDefaultNumberOfTemporalLayers = 1; // 1:1 108 109 static const int kTimestampDeltaInSecondsForWarning = 2; 110 111 static const int kMaxExternalVideoCodecs = 8; 112 static const int kExternalVideoPayloadTypeBase = 120; 113 114 // Static allocation of payload type values for external video codec. 115 static int GetExternalVideoPayloadType(int index) { 116 ASSERT(index >= 0 && index < kMaxExternalVideoCodecs); 117 return kExternalVideoPayloadTypeBase + index; 118 } 119 120 static void LogMultiline(talk_base::LoggingSeverity sev, char* text) { 121 const char* delim = "\r\n"; 122 // TODO(fbarchard): Fix strtok lint warning. 123 for (char* tok = strtok(text, delim); tok; tok = strtok(NULL, delim)) { 124 LOG_V(sev) << tok; 125 } 126 } 127 128 // Severity is an integer because it comes is assumed to be from command line. 129 static int SeverityToFilter(int severity) { 130 int filter = webrtc::kTraceNone; 131 switch (severity) { 132 case talk_base::LS_VERBOSE: 133 filter |= webrtc::kTraceAll; 134 case talk_base::LS_INFO: 135 filter |= (webrtc::kTraceStateInfo | webrtc::kTraceInfo); 136 case talk_base::LS_WARNING: 137 filter |= (webrtc::kTraceTerseInfo | webrtc::kTraceWarning); 138 case talk_base::LS_ERROR: 139 filter |= (webrtc::kTraceError | webrtc::kTraceCritical); 140 } 141 return filter; 142 } 143 144 static const int kCpuMonitorPeriodMs = 2000; // 2 seconds. 145 146 static const bool kNotSending = false; 147 148 // Extension header for RTP timestamp offset, see RFC 5450 for details: 149 // http://tools.ietf.org/html/rfc5450 150 static const char kRtpTimestampOffsetHeaderExtension[] = 151 "urn:ietf:params:rtp-hdrext:toffset"; 152 static const int kRtpTimeOffsetExtensionId = 2; 153 154 // Extension header for absolute send time, see url for details: 155 // http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time 156 static const char kRtpAbsoluteSendTimeHeaderExtension[] = 157 "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"; 158 static const int kRtpAbsoluteSendTimeExtensionId = 3; 159 // Default video dscp value. 160 // See http://tools.ietf.org/html/rfc2474 for details 161 // See also http://tools.ietf.org/html/draft-jennings-rtcweb-qos-00 162 static const talk_base::DiffServCodePoint kVideoDscpValue = 163 talk_base::DSCP_AF41; 164 165 static bool IsNackEnabled(const VideoCodec& codec) { 166 return codec.HasFeedbackParam(FeedbackParam(kRtcpFbParamNack, 167 kParamValueEmpty)); 168 } 169 170 // Returns true if Receiver Estimated Max Bitrate is enabled. 171 static bool IsRembEnabled(const VideoCodec& codec) { 172 return codec.HasFeedbackParam(FeedbackParam(kRtcpFbParamRemb, 173 kParamValueEmpty)); 174 } 175 176 struct FlushBlackFrameData : public talk_base::MessageData { 177 FlushBlackFrameData(uint32 s, int64 t) : ssrc(s), timestamp(t) { 178 } 179 uint32 ssrc; 180 int64 timestamp; 181 }; 182 183 class WebRtcRenderAdapter : public webrtc::ExternalRenderer { 184 public: 185 explicit WebRtcRenderAdapter(VideoRenderer* renderer) 186 : renderer_(renderer), width_(0), height_(0), watermark_enabled_(false) { 187 } 188 189 virtual ~WebRtcRenderAdapter() { 190 } 191 192 void set_watermark_enabled(bool enable) { 193 talk_base::CritScope cs(&crit_); 194 watermark_enabled_ = enable; 195 } 196 197 void SetRenderer(VideoRenderer* renderer) { 198 talk_base::CritScope cs(&crit_); 199 renderer_ = renderer; 200 // FrameSizeChange may have already been called when renderer was not set. 201 // If so we should call SetSize here. 202 // TODO(ronghuawu): Add unit test for this case. Didn't do it now 203 // because the WebRtcRenderAdapter is currently hiding in cc file. No 204 // good way to get access to it from the unit test. 205 if (width_ > 0 && height_ > 0 && renderer_ != NULL) { 206 if (!renderer_->SetSize(width_, height_, 0)) { 207 LOG(LS_ERROR) 208 << "WebRtcRenderAdapter SetRenderer failed to SetSize to: " 209 << width_ << "x" << height_; 210 } 211 } 212 } 213 214 // Implementation of webrtc::ExternalRenderer. 215 virtual int FrameSizeChange(unsigned int width, unsigned int height, 216 unsigned int /*number_of_streams*/) { 217 talk_base::CritScope cs(&crit_); 218 width_ = width; 219 height_ = height; 220 LOG(LS_INFO) << "WebRtcRenderAdapter frame size changed to: " 221 << width << "x" << height; 222 if (renderer_ == NULL) { 223 LOG(LS_VERBOSE) << "WebRtcRenderAdapter the renderer has not been set. " 224 << "SetSize will be called later in SetRenderer."; 225 return 0; 226 } 227 return renderer_->SetSize(width_, height_, 0) ? 0 : -1; 228 } 229 230 virtual int DeliverFrame(unsigned char* buffer, int buffer_size, 231 uint32_t time_stamp, int64_t render_time, 232 void* handle) { 233 talk_base::CritScope cs(&crit_); 234 frame_rate_tracker_.Update(1); 235 if (renderer_ == NULL) { 236 return 0; 237 } 238 // Convert 90K rtp timestamp to ns timestamp. 239 int64 rtp_time_stamp_in_ns = (time_stamp / 90) * 240 talk_base::kNumNanosecsPerMillisec; 241 // Convert milisecond render time to ns timestamp. 242 int64 render_time_stamp_in_ns = render_time * 243 talk_base::kNumNanosecsPerMillisec; 244 // Send the rtp timestamp to renderer as the VideoFrame timestamp. 245 // and the render timestamp as the VideoFrame elapsed_time. 246 if (handle == NULL) { 247 return DeliverBufferFrame(buffer, buffer_size, render_time_stamp_in_ns, 248 rtp_time_stamp_in_ns); 249 } else { 250 return DeliverTextureFrame(handle, render_time_stamp_in_ns, 251 rtp_time_stamp_in_ns); 252 } 253 } 254 255 virtual bool IsTextureSupported() { return true; } 256 257 int DeliverBufferFrame(unsigned char* buffer, int buffer_size, 258 int64 elapsed_time, int64 time_stamp) { 259 WebRtcVideoFrame video_frame; 260 video_frame.Alias(buffer, buffer_size, width_, height_, 261 1, 1, elapsed_time, time_stamp, 0); 262 263 264 // Sanity check on decoded frame size. 265 if (buffer_size != static_cast<int>(VideoFrame::SizeOf(width_, height_))) { 266 LOG(LS_WARNING) << "WebRtcRenderAdapter received a strange frame size: " 267 << buffer_size; 268 } 269 270 int ret = renderer_->RenderFrame(&video_frame) ? 0 : -1; 271 return ret; 272 } 273 274 int DeliverTextureFrame(void* handle, int64 elapsed_time, int64 time_stamp) { 275 WebRtcTextureVideoFrame video_frame( 276 static_cast<webrtc::NativeHandle*>(handle), width_, height_, 277 elapsed_time, time_stamp); 278 return renderer_->RenderFrame(&video_frame); 279 } 280 281 unsigned int width() { 282 talk_base::CritScope cs(&crit_); 283 return width_; 284 } 285 286 unsigned int height() { 287 talk_base::CritScope cs(&crit_); 288 return height_; 289 } 290 291 int framerate() { 292 talk_base::CritScope cs(&crit_); 293 return static_cast<int>(frame_rate_tracker_.units_second()); 294 } 295 296 VideoRenderer* renderer() { 297 talk_base::CritScope cs(&crit_); 298 return renderer_; 299 } 300 301 private: 302 talk_base::CriticalSection crit_; 303 VideoRenderer* renderer_; 304 unsigned int width_; 305 unsigned int height_; 306 talk_base::RateTracker frame_rate_tracker_; 307 bool watermark_enabled_; 308 }; 309 310 class WebRtcDecoderObserver : public webrtc::ViEDecoderObserver { 311 public: 312 explicit WebRtcDecoderObserver(int video_channel) 313 : video_channel_(video_channel), 314 framerate_(0), 315 bitrate_(0), 316 decode_ms_(0), 317 max_decode_ms_(0), 318 current_delay_ms_(0), 319 target_delay_ms_(0), 320 jitter_buffer_ms_(0), 321 min_playout_delay_ms_(0), 322 render_delay_ms_(0), 323 firs_requested_(0) { 324 } 325 326 // virtual functions from VieDecoderObserver. 327 virtual void IncomingCodecChanged(const int videoChannel, 328 const webrtc::VideoCodec& videoCodec) {} 329 virtual void IncomingRate(const int videoChannel, 330 const unsigned int framerate, 331 const unsigned int bitrate) { 332 talk_base::CritScope cs(&crit_); 333 ASSERT(video_channel_ == videoChannel); 334 framerate_ = framerate; 335 bitrate_ = bitrate; 336 } 337 338 virtual void DecoderTiming(int decode_ms, 339 int max_decode_ms, 340 int current_delay_ms, 341 int target_delay_ms, 342 int jitter_buffer_ms, 343 int min_playout_delay_ms, 344 int render_delay_ms) { 345 talk_base::CritScope cs(&crit_); 346 decode_ms_ = decode_ms; 347 max_decode_ms_ = max_decode_ms; 348 current_delay_ms_ = current_delay_ms; 349 target_delay_ms_ = target_delay_ms; 350 jitter_buffer_ms_ = jitter_buffer_ms; 351 min_playout_delay_ms_ = min_playout_delay_ms; 352 render_delay_ms_ = render_delay_ms; 353 } 354 355 virtual void RequestNewKeyFrame(const int videoChannel) { 356 talk_base::CritScope cs(&crit_); 357 ASSERT(video_channel_ == videoChannel); 358 ++firs_requested_; 359 } 360 361 // Populate |rinfo| based on previously-set data in |*this|. 362 void ExportTo(VideoReceiverInfo* rinfo) { 363 talk_base::CritScope cs(&crit_); 364 rinfo->firs_sent = firs_requested_; 365 rinfo->framerate_rcvd = framerate_; 366 rinfo->decode_ms = decode_ms_; 367 rinfo->max_decode_ms = max_decode_ms_; 368 rinfo->current_delay_ms = current_delay_ms_; 369 rinfo->target_delay_ms = target_delay_ms_; 370 rinfo->jitter_buffer_ms = jitter_buffer_ms_; 371 rinfo->min_playout_delay_ms = min_playout_delay_ms_; 372 rinfo->render_delay_ms = render_delay_ms_; 373 } 374 375 private: 376 mutable talk_base::CriticalSection crit_; 377 int video_channel_; 378 int framerate_; 379 int bitrate_; 380 int decode_ms_; 381 int max_decode_ms_; 382 int current_delay_ms_; 383 int target_delay_ms_; 384 int jitter_buffer_ms_; 385 int min_playout_delay_ms_; 386 int render_delay_ms_; 387 int firs_requested_; 388 }; 389 390 class WebRtcEncoderObserver : public webrtc::ViEEncoderObserver { 391 public: 392 explicit WebRtcEncoderObserver(int video_channel) 393 : video_channel_(video_channel), 394 framerate_(0), 395 bitrate_(0), 396 suspended_(false) { 397 } 398 399 // virtual functions from VieEncoderObserver. 400 virtual void OutgoingRate(const int videoChannel, 401 const unsigned int framerate, 402 const unsigned int bitrate) { 403 talk_base::CritScope cs(&crit_); 404 ASSERT(video_channel_ == videoChannel); 405 framerate_ = framerate; 406 bitrate_ = bitrate; 407 } 408 409 virtual void SuspendChange(int video_channel, bool is_suspended) { 410 talk_base::CritScope cs(&crit_); 411 ASSERT(video_channel_ == video_channel); 412 suspended_ = is_suspended; 413 } 414 415 int framerate() const { 416 talk_base::CritScope cs(&crit_); 417 return framerate_; 418 } 419 int bitrate() const { 420 talk_base::CritScope cs(&crit_); 421 return bitrate_; 422 } 423 bool suspended() const { 424 talk_base::CritScope cs(&crit_); 425 return suspended_; 426 } 427 428 private: 429 mutable talk_base::CriticalSection crit_; 430 int video_channel_; 431 int framerate_; 432 int bitrate_; 433 bool suspended_; 434 }; 435 436 class WebRtcLocalStreamInfo { 437 public: 438 WebRtcLocalStreamInfo() 439 : width_(0), height_(0), elapsed_time_(-1), time_stamp_(-1) {} 440 size_t width() const { 441 talk_base::CritScope cs(&crit_); 442 return width_; 443 } 444 size_t height() const { 445 talk_base::CritScope cs(&crit_); 446 return height_; 447 } 448 int64 elapsed_time() const { 449 talk_base::CritScope cs(&crit_); 450 return elapsed_time_; 451 } 452 int64 time_stamp() const { 453 talk_base::CritScope cs(&crit_); 454 return time_stamp_; 455 } 456 int framerate() { 457 talk_base::CritScope cs(&crit_); 458 return static_cast<int>(rate_tracker_.units_second()); 459 } 460 void GetLastFrameInfo( 461 size_t* width, size_t* height, int64* elapsed_time) const { 462 talk_base::CritScope cs(&crit_); 463 *width = width_; 464 *height = height_; 465 *elapsed_time = elapsed_time_; 466 } 467 468 void UpdateFrame(const VideoFrame* frame) { 469 talk_base::CritScope cs(&crit_); 470 471 width_ = frame->GetWidth(); 472 height_ = frame->GetHeight(); 473 elapsed_time_ = frame->GetElapsedTime(); 474 time_stamp_ = frame->GetTimeStamp(); 475 476 rate_tracker_.Update(1); 477 } 478 479 private: 480 mutable talk_base::CriticalSection crit_; 481 size_t width_; 482 size_t height_; 483 int64 elapsed_time_; 484 int64 time_stamp_; 485 talk_base::RateTracker rate_tracker_; 486 487 DISALLOW_COPY_AND_ASSIGN(WebRtcLocalStreamInfo); 488 }; 489 490 // WebRtcVideoChannelRecvInfo is a container class with members such as renderer 491 // and a decoder observer that is used by receive channels. 492 // It must exist as long as the receive channel is connected to renderer or a 493 // decoder observer in this class and methods in the class should only be called 494 // from the worker thread. 495 class WebRtcVideoChannelRecvInfo { 496 public: 497 typedef std::map<int, webrtc::VideoDecoder*> DecoderMap; // key: payload type 498 explicit WebRtcVideoChannelRecvInfo(int channel_id) 499 : channel_id_(channel_id), 500 render_adapter_(NULL), 501 decoder_observer_(channel_id) { 502 } 503 int channel_id() { return channel_id_; } 504 void SetRenderer(VideoRenderer* renderer) { 505 render_adapter_.SetRenderer(renderer); 506 } 507 WebRtcRenderAdapter* render_adapter() { return &render_adapter_; } 508 WebRtcDecoderObserver* decoder_observer() { return &decoder_observer_; } 509 void RegisterDecoder(int pl_type, webrtc::VideoDecoder* decoder) { 510 ASSERT(!IsDecoderRegistered(pl_type)); 511 registered_decoders_[pl_type] = decoder; 512 } 513 bool IsDecoderRegistered(int pl_type) { 514 return registered_decoders_.count(pl_type) != 0; 515 } 516 const DecoderMap& registered_decoders() { 517 return registered_decoders_; 518 } 519 void ClearRegisteredDecoders() { 520 registered_decoders_.clear(); 521 } 522 523 private: 524 int channel_id_; // Webrtc video channel number. 525 // Renderer for this channel. 526 WebRtcRenderAdapter render_adapter_; 527 WebRtcDecoderObserver decoder_observer_; 528 DecoderMap registered_decoders_; 529 }; 530 531 class WebRtcOveruseObserver : public webrtc::CpuOveruseObserver { 532 public: 533 explicit WebRtcOveruseObserver(CoordinatedVideoAdapter* video_adapter) 534 : video_adapter_(video_adapter), 535 enabled_(false) { 536 } 537 538 // TODO(mflodman): Consider sending resolution as part of event, to let 539 // adapter know what resolution the request is based on. Helps eliminate stale 540 // data, race conditions. 541 virtual void OveruseDetected() OVERRIDE { 542 talk_base::CritScope cs(&crit_); 543 if (!enabled_) { 544 return; 545 } 546 547 video_adapter_->OnCpuResolutionRequest(CoordinatedVideoAdapter::DOWNGRADE); 548 } 549 550 virtual void NormalUsage() OVERRIDE { 551 talk_base::CritScope cs(&crit_); 552 if (!enabled_) { 553 return; 554 } 555 556 video_adapter_->OnCpuResolutionRequest(CoordinatedVideoAdapter::UPGRADE); 557 } 558 559 void Enable(bool enable) { 560 talk_base::CritScope cs(&crit_); 561 enabled_ = enable; 562 } 563 564 private: 565 CoordinatedVideoAdapter* video_adapter_; 566 bool enabled_; 567 talk_base::CriticalSection crit_; 568 }; 569 570 571 class WebRtcVideoChannelSendInfo : public sigslot::has_slots<> { 572 public: 573 typedef std::map<int, webrtc::VideoEncoder*> EncoderMap; // key: payload type 574 WebRtcVideoChannelSendInfo(int channel_id, int capture_id, 575 webrtc::ViEExternalCapture* external_capture, 576 talk_base::CpuMonitor* cpu_monitor) 577 : channel_id_(channel_id), 578 capture_id_(capture_id), 579 sending_(false), 580 muted_(false), 581 video_capturer_(NULL), 582 encoder_observer_(channel_id), 583 external_capture_(external_capture), 584 capturer_updated_(false), 585 interval_(0), 586 video_adapter_(new CoordinatedVideoAdapter), 587 cpu_monitor_(cpu_monitor) { 588 overuse_observer_.reset(new WebRtcOveruseObserver(video_adapter_.get())); 589 SignalCpuAdaptationUnable.repeat(video_adapter_->SignalCpuAdaptationUnable); 590 if (cpu_monitor) { 591 cpu_monitor->SignalUpdate.connect( 592 video_adapter_.get(), &CoordinatedVideoAdapter::OnCpuLoadUpdated); 593 } 594 } 595 596 int channel_id() const { return channel_id_; } 597 int capture_id() const { return capture_id_; } 598 void set_sending(bool sending) { sending_ = sending; } 599 bool sending() const { return sending_; } 600 void set_muted(bool on) { 601 // TODO(asapersson): add support. 602 // video_adapter_->SetBlackOutput(on); 603 muted_ = on; 604 } 605 bool muted() {return muted_; } 606 607 WebRtcEncoderObserver* encoder_observer() { return &encoder_observer_; } 608 webrtc::ViEExternalCapture* external_capture() { return external_capture_; } 609 const VideoFormat& video_format() const { 610 return video_format_; 611 } 612 void set_video_format(const VideoFormat& video_format) { 613 video_format_ = video_format; 614 if (video_format_ != cricket::VideoFormat()) { 615 interval_ = video_format_.interval; 616 } 617 video_adapter_->OnOutputFormatRequest(video_format_); 618 } 619 void set_interval(int64 interval) { 620 if (video_format() == cricket::VideoFormat()) { 621 interval_ = interval; 622 } 623 } 624 int64 interval() { return interval_; } 625 626 void InitializeAdapterOutputFormat(const webrtc::VideoCodec& codec) { 627 VideoFormat format(codec.width, codec.height, 628 VideoFormat::FpsToInterval(codec.maxFramerate), 629 FOURCC_I420); 630 if (video_adapter_->output_format().IsSize0x0()) { 631 video_adapter_->SetOutputFormat(format); 632 } 633 } 634 635 int CurrentAdaptReason() const { 636 return video_adapter_->adapt_reason(); 637 } 638 webrtc::CpuOveruseObserver* overuse_observer() { 639 return overuse_observer_.get(); 640 } 641 642 StreamParams* stream_params() { return stream_params_.get(); } 643 void set_stream_params(const StreamParams& sp) { 644 stream_params_.reset(new StreamParams(sp)); 645 } 646 void ClearStreamParams() { stream_params_.reset(); } 647 bool has_ssrc(uint32 local_ssrc) const { 648 return !stream_params_ ? false : 649 stream_params_->has_ssrc(local_ssrc); 650 } 651 WebRtcLocalStreamInfo* local_stream_info() { 652 return &local_stream_info_; 653 } 654 VideoCapturer* video_capturer() { 655 return video_capturer_; 656 } 657 void set_video_capturer(VideoCapturer* video_capturer) { 658 if (video_capturer == video_capturer_) { 659 return; 660 } 661 capturer_updated_ = true; 662 video_capturer_ = video_capturer; 663 if (video_capturer && !video_capturer->IsScreencast()) { 664 const VideoFormat* capture_format = video_capturer->GetCaptureFormat(); 665 if (capture_format) { 666 // TODO(thorcarpenter): This is broken. Video capturer doesn't have 667 // a capture format until the capturer is started. So, if 668 // the capturer is started immediately after calling set_video_capturer 669 // video adapter may not have the input format set, the interval may 670 // be zero, and all frames may be dropped. 671 // Consider fixing this by having video_adapter keep a pointer to the 672 // video capturer. 673 video_adapter_->SetInputFormat(*capture_format); 674 } 675 // TODO(thorcarpenter): When the adapter supports "only frame dropping" 676 // mode, also hook it up to screencast capturers. 677 video_capturer->SignalAdaptFrame.connect( 678 this, &WebRtcVideoChannelSendInfo::AdaptFrame); 679 } 680 } 681 682 void AdaptFrame(VideoCapturer* capturer, const VideoFrame* input, 683 VideoFrame** adapted) { 684 video_adapter_->AdaptFrame(input, adapted); 685 } 686 687 void ApplyCpuOptions(const VideoOptions& options) { 688 bool cpu_adapt, cpu_smoothing, adapt_third; 689 float low, med, high; 690 if (options.adapt_input_to_cpu_usage.Get(&cpu_adapt)) { 691 video_adapter_->set_cpu_adaptation(cpu_adapt); 692 } 693 if (options.adapt_cpu_with_smoothing.Get(&cpu_smoothing)) { 694 video_adapter_->set_cpu_smoothing(cpu_smoothing); 695 } 696 if (options.process_adaptation_threshhold.Get(&med)) { 697 video_adapter_->set_process_threshold(med); 698 } 699 if (options.system_low_adaptation_threshhold.Get(&low)) { 700 video_adapter_->set_low_system_threshold(low); 701 } 702 if (options.system_high_adaptation_threshhold.Get(&high)) { 703 video_adapter_->set_high_system_threshold(high); 704 } 705 if (options.video_adapt_third.Get(&adapt_third)) { 706 video_adapter_->set_scale_third(adapt_third); 707 } 708 } 709 710 void SetCpuOveruseDetection(bool enable) { 711 if (cpu_monitor_ && enable) { 712 cpu_monitor_->SignalUpdate.disconnect(video_adapter_.get()); 713 } 714 overuse_observer_->Enable(enable); 715 video_adapter_->set_cpu_adaptation(enable); 716 } 717 718 void ProcessFrame(const VideoFrame& original_frame, bool mute, 719 VideoFrame** processed_frame) { 720 if (!mute) { 721 *processed_frame = original_frame.Copy(); 722 } else { 723 WebRtcVideoFrame* black_frame = new WebRtcVideoFrame(); 724 black_frame->InitToBlack(static_cast<int>(original_frame.GetWidth()), 725 static_cast<int>(original_frame.GetHeight()), 726 1, 1, 727 original_frame.GetElapsedTime(), 728 original_frame.GetTimeStamp()); 729 *processed_frame = black_frame; 730 } 731 local_stream_info_.UpdateFrame(*processed_frame); 732 } 733 void RegisterEncoder(int pl_type, webrtc::VideoEncoder* encoder) { 734 ASSERT(!IsEncoderRegistered(pl_type)); 735 registered_encoders_[pl_type] = encoder; 736 } 737 bool IsEncoderRegistered(int pl_type) { 738 return registered_encoders_.count(pl_type) != 0; 739 } 740 const EncoderMap& registered_encoders() { 741 return registered_encoders_; 742 } 743 void ClearRegisteredEncoders() { 744 registered_encoders_.clear(); 745 } 746 747 sigslot::repeater0<> SignalCpuAdaptationUnable; 748 749 private: 750 int channel_id_; 751 int capture_id_; 752 bool sending_; 753 bool muted_; 754 VideoCapturer* video_capturer_; 755 WebRtcEncoderObserver encoder_observer_; 756 webrtc::ViEExternalCapture* external_capture_; 757 EncoderMap registered_encoders_; 758 759 VideoFormat video_format_; 760 761 talk_base::scoped_ptr<StreamParams> stream_params_; 762 763 WebRtcLocalStreamInfo local_stream_info_; 764 765 bool capturer_updated_; 766 767 int64 interval_; 768 769 talk_base::scoped_ptr<CoordinatedVideoAdapter> video_adapter_; 770 talk_base::CpuMonitor* cpu_monitor_; 771 talk_base::scoped_ptr<WebRtcOveruseObserver> overuse_observer_; 772 }; 773 774 const WebRtcVideoEngine::VideoCodecPref 775 WebRtcVideoEngine::kVideoCodecPrefs[] = { 776 {kVp8PayloadName, 100, -1, 0}, 777 {kRedPayloadName, 116, -1, 1}, 778 {kFecPayloadName, 117, -1, 2}, 779 {kRtxCodecName, 96, 100, 3}, 780 }; 781 782 // The formats are sorted by the descending order of width. We use the order to 783 // find the next format for CPU and bandwidth adaptation. 784 const VideoFormatPod WebRtcVideoEngine::kVideoFormats[] = { 785 {1280, 800, FPS_TO_INTERVAL(30), FOURCC_ANY}, 786 {1280, 720, FPS_TO_INTERVAL(30), FOURCC_ANY}, 787 {960, 600, FPS_TO_INTERVAL(30), FOURCC_ANY}, 788 {960, 540, FPS_TO_INTERVAL(30), FOURCC_ANY}, 789 {640, 400, FPS_TO_INTERVAL(30), FOURCC_ANY}, 790 {640, 360, FPS_TO_INTERVAL(30), FOURCC_ANY}, 791 {640, 480, FPS_TO_INTERVAL(30), FOURCC_ANY}, 792 {480, 300, FPS_TO_INTERVAL(30), FOURCC_ANY}, 793 {480, 270, FPS_TO_INTERVAL(30), FOURCC_ANY}, 794 {480, 360, FPS_TO_INTERVAL(30), FOURCC_ANY}, 795 {320, 200, FPS_TO_INTERVAL(30), FOURCC_ANY}, 796 {320, 180, FPS_TO_INTERVAL(30), FOURCC_ANY}, 797 {320, 240, FPS_TO_INTERVAL(30), FOURCC_ANY}, 798 {240, 150, FPS_TO_INTERVAL(30), FOURCC_ANY}, 799 {240, 135, FPS_TO_INTERVAL(30), FOURCC_ANY}, 800 {240, 180, FPS_TO_INTERVAL(30), FOURCC_ANY}, 801 {160, 100, FPS_TO_INTERVAL(30), FOURCC_ANY}, 802 {160, 90, FPS_TO_INTERVAL(30), FOURCC_ANY}, 803 {160, 120, FPS_TO_INTERVAL(30), FOURCC_ANY}, 804 }; 805 806 const VideoFormatPod WebRtcVideoEngine::kDefaultVideoFormat = 807 {640, 400, FPS_TO_INTERVAL(30), FOURCC_ANY}; 808 809 static void UpdateVideoCodec(const cricket::VideoFormat& video_format, 810 webrtc::VideoCodec* target_codec) { 811 if ((target_codec == NULL) || (video_format == cricket::VideoFormat())) { 812 return; 813 } 814 target_codec->width = video_format.width; 815 target_codec->height = video_format.height; 816 target_codec->maxFramerate = cricket::VideoFormat::IntervalToFps( 817 video_format.interval); 818 } 819 820 WebRtcVideoEngine::WebRtcVideoEngine() { 821 Construct(new ViEWrapper(), new ViETraceWrapper(), NULL, 822 new talk_base::CpuMonitor(NULL)); 823 } 824 825 WebRtcVideoEngine::WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine, 826 ViEWrapper* vie_wrapper, 827 talk_base::CpuMonitor* cpu_monitor) { 828 Construct(vie_wrapper, new ViETraceWrapper(), voice_engine, cpu_monitor); 829 } 830 831 WebRtcVideoEngine::WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine, 832 ViEWrapper* vie_wrapper, 833 ViETraceWrapper* tracing, 834 talk_base::CpuMonitor* cpu_monitor) { 835 Construct(vie_wrapper, tracing, voice_engine, cpu_monitor); 836 } 837 838 void WebRtcVideoEngine::Construct(ViEWrapper* vie_wrapper, 839 ViETraceWrapper* tracing, 840 WebRtcVoiceEngine* voice_engine, 841 talk_base::CpuMonitor* cpu_monitor) { 842 LOG(LS_INFO) << "WebRtcVideoEngine::WebRtcVideoEngine"; 843 worker_thread_ = NULL; 844 vie_wrapper_.reset(vie_wrapper); 845 vie_wrapper_base_initialized_ = false; 846 tracing_.reset(tracing); 847 voice_engine_ = voice_engine; 848 initialized_ = false; 849 SetTraceFilter(SeverityToFilter(kDefaultLogSeverity)); 850 render_module_.reset(new WebRtcPassthroughRender()); 851 local_renderer_w_ = local_renderer_h_ = 0; 852 local_renderer_ = NULL; 853 capture_started_ = false; 854 decoder_factory_ = NULL; 855 encoder_factory_ = NULL; 856 cpu_monitor_.reset(cpu_monitor); 857 858 SetTraceOptions(""); 859 if (tracing_->SetTraceCallback(this) != 0) { 860 LOG_RTCERR1(SetTraceCallback, this); 861 } 862 863 // Set default quality levels for our supported codecs. We override them here 864 // if we know your cpu performance is low, and they can be updated explicitly 865 // by calling SetDefaultCodec. For example by a flute preference setting, or 866 // by the server with a jec in response to our reported system info. 867 VideoCodec max_codec(kVideoCodecPrefs[0].payload_type, 868 kVideoCodecPrefs[0].name, 869 kDefaultVideoFormat.width, 870 kDefaultVideoFormat.height, 871 VideoFormat::IntervalToFps(kDefaultVideoFormat.interval), 872 0); 873 if (!SetDefaultCodec(max_codec)) { 874 LOG(LS_ERROR) << "Failed to initialize list of supported codec types"; 875 } 876 877 878 // Load our RTP Header extensions. 879 rtp_header_extensions_.push_back( 880 RtpHeaderExtension(kRtpTimestampOffsetHeaderExtension, 881 kRtpTimeOffsetExtensionId)); 882 rtp_header_extensions_.push_back( 883 RtpHeaderExtension(kRtpAbsoluteSendTimeHeaderExtension, 884 kRtpAbsoluteSendTimeExtensionId)); 885 } 886 887 WebRtcVideoEngine::~WebRtcVideoEngine() { 888 LOG(LS_INFO) << "WebRtcVideoEngine::~WebRtcVideoEngine"; 889 if (initialized_) { 890 Terminate(); 891 } 892 if (encoder_factory_) { 893 encoder_factory_->RemoveObserver(this); 894 } 895 tracing_->SetTraceCallback(NULL); 896 // Test to see if the media processor was deregistered properly. 897 ASSERT(SignalMediaFrame.is_empty()); 898 } 899 900 bool WebRtcVideoEngine::Init(talk_base::Thread* worker_thread) { 901 LOG(LS_INFO) << "WebRtcVideoEngine::Init"; 902 worker_thread_ = worker_thread; 903 ASSERT(worker_thread_ != NULL); 904 905 cpu_monitor_->set_thread(worker_thread_); 906 if (!cpu_monitor_->Start(kCpuMonitorPeriodMs)) { 907 LOG(LS_ERROR) << "Failed to start CPU monitor."; 908 cpu_monitor_.reset(); 909 } 910 911 bool result = InitVideoEngine(); 912 if (result) { 913 LOG(LS_INFO) << "VideoEngine Init done"; 914 } else { 915 LOG(LS_ERROR) << "VideoEngine Init failed, releasing"; 916 Terminate(); 917 } 918 return result; 919 } 920 921 bool WebRtcVideoEngine::InitVideoEngine() { 922 LOG(LS_INFO) << "WebRtcVideoEngine::InitVideoEngine"; 923 924 // Init WebRTC VideoEngine. 925 if (!vie_wrapper_base_initialized_) { 926 if (vie_wrapper_->base()->Init() != 0) { 927 LOG_RTCERR0(Init); 928 return false; 929 } 930 vie_wrapper_base_initialized_ = true; 931 } 932 933 // Log the VoiceEngine version info. 934 char buffer[1024] = ""; 935 if (vie_wrapper_->base()->GetVersion(buffer) != 0) { 936 LOG_RTCERR0(GetVersion); 937 return false; 938 } 939 940 LOG(LS_INFO) << "WebRtc VideoEngine Version:"; 941 LogMultiline(talk_base::LS_INFO, buffer); 942 943 // Hook up to VoiceEngine for sync purposes, if supplied. 944 if (!voice_engine_) { 945 LOG(LS_WARNING) << "NULL voice engine"; 946 } else if ((vie_wrapper_->base()->SetVoiceEngine( 947 voice_engine_->voe()->engine())) != 0) { 948 LOG_RTCERR0(SetVoiceEngine); 949 return false; 950 } 951 952 // Register our custom render module. 953 if (vie_wrapper_->render()->RegisterVideoRenderModule( 954 *render_module_.get()) != 0) { 955 LOG_RTCERR0(RegisterVideoRenderModule); 956 return false; 957 } 958 959 initialized_ = true; 960 return true; 961 } 962 963 void WebRtcVideoEngine::Terminate() { 964 LOG(LS_INFO) << "WebRtcVideoEngine::Terminate"; 965 initialized_ = false; 966 967 if (vie_wrapper_->render()->DeRegisterVideoRenderModule( 968 *render_module_.get()) != 0) { 969 LOG_RTCERR0(DeRegisterVideoRenderModule); 970 } 971 972 if (vie_wrapper_->base()->SetVoiceEngine(NULL) != 0) { 973 LOG_RTCERR0(SetVoiceEngine); 974 } 975 976 cpu_monitor_->Stop(); 977 } 978 979 int WebRtcVideoEngine::GetCapabilities() { 980 return VIDEO_RECV | VIDEO_SEND; 981 } 982 983 bool WebRtcVideoEngine::SetOptions(const VideoOptions &options) { 984 return true; 985 } 986 987 bool WebRtcVideoEngine::SetDefaultEncoderConfig( 988 const VideoEncoderConfig& config) { 989 return SetDefaultCodec(config.max_codec); 990 } 991 992 VideoEncoderConfig WebRtcVideoEngine::GetDefaultEncoderConfig() const { 993 ASSERT(!video_codecs_.empty()); 994 VideoCodec max_codec(kVideoCodecPrefs[0].payload_type, 995 kVideoCodecPrefs[0].name, 996 video_codecs_[0].width, 997 video_codecs_[0].height, 998 video_codecs_[0].framerate, 999 0); 1000 return VideoEncoderConfig(max_codec); 1001 } 1002 1003 // SetDefaultCodec may be called while the capturer is running. For example, a 1004 // test call is started in a page with QVGA default codec, and then a real call 1005 // is started in another page with VGA default codec. This is the corner case 1006 // and happens only when a session is started. We ignore this case currently. 1007 bool WebRtcVideoEngine::SetDefaultCodec(const VideoCodec& codec) { 1008 if (!RebuildCodecList(codec)) { 1009 LOG(LS_WARNING) << "Failed to RebuildCodecList"; 1010 return false; 1011 } 1012 1013 ASSERT(!video_codecs_.empty()); 1014 default_codec_format_ = VideoFormat( 1015 video_codecs_[0].width, 1016 video_codecs_[0].height, 1017 VideoFormat::FpsToInterval(video_codecs_[0].framerate), 1018 FOURCC_ANY); 1019 return true; 1020 } 1021 1022 WebRtcVideoMediaChannel* WebRtcVideoEngine::CreateChannel( 1023 VoiceMediaChannel* voice_channel) { 1024 WebRtcVideoMediaChannel* channel = 1025 new WebRtcVideoMediaChannel(this, voice_channel); 1026 if (!channel->Init()) { 1027 delete channel; 1028 channel = NULL; 1029 } 1030 return channel; 1031 } 1032 1033 bool WebRtcVideoEngine::SetLocalRenderer(VideoRenderer* renderer) { 1034 local_renderer_w_ = local_renderer_h_ = 0; 1035 local_renderer_ = renderer; 1036 return true; 1037 } 1038 1039 const std::vector<VideoCodec>& WebRtcVideoEngine::codecs() const { 1040 return video_codecs_; 1041 } 1042 1043 const std::vector<RtpHeaderExtension>& 1044 WebRtcVideoEngine::rtp_header_extensions() const { 1045 return rtp_header_extensions_; 1046 } 1047 1048 void WebRtcVideoEngine::SetLogging(int min_sev, const char* filter) { 1049 // if min_sev == -1, we keep the current log level. 1050 if (min_sev >= 0) { 1051 SetTraceFilter(SeverityToFilter(min_sev)); 1052 } 1053 SetTraceOptions(filter); 1054 } 1055 1056 int WebRtcVideoEngine::GetLastEngineError() { 1057 return vie_wrapper_->error(); 1058 } 1059 1060 // Checks to see whether we comprehend and could receive a particular codec 1061 bool WebRtcVideoEngine::FindCodec(const VideoCodec& in) { 1062 for (int i = 0; i < ARRAY_SIZE(kVideoFormats); ++i) { 1063 const VideoFormat fmt(kVideoFormats[i]); 1064 if ((in.width == 0 && in.height == 0) || 1065 (fmt.width == in.width && fmt.height == in.height)) { 1066 if (encoder_factory_) { 1067 const std::vector<WebRtcVideoEncoderFactory::VideoCodec>& codecs = 1068 encoder_factory_->codecs(); 1069 for (size_t j = 0; j < codecs.size(); ++j) { 1070 VideoCodec codec(GetExternalVideoPayloadType(static_cast<int>(j)), 1071 codecs[j].name, 0, 0, 0, 0); 1072 if (codec.Matches(in)) 1073 return true; 1074 } 1075 } 1076 for (size_t j = 0; j < ARRAY_SIZE(kVideoCodecPrefs); ++j) { 1077 VideoCodec codec(kVideoCodecPrefs[j].payload_type, 1078 kVideoCodecPrefs[j].name, 0, 0, 0, 0); 1079 if (codec.Matches(in)) { 1080 return true; 1081 } 1082 } 1083 } 1084 } 1085 return false; 1086 } 1087 1088 // Given the requested codec, returns true if we can send that codec type and 1089 // updates out with the best quality we could send for that codec. If current is 1090 // not empty, we constrain out so that its aspect ratio matches current's. 1091 bool WebRtcVideoEngine::CanSendCodec(const VideoCodec& requested, 1092 const VideoCodec& current, 1093 VideoCodec* out) { 1094 if (!out) { 1095 return false; 1096 } 1097 1098 std::vector<VideoCodec>::const_iterator local_max; 1099 for (local_max = video_codecs_.begin(); 1100 local_max < video_codecs_.end(); 1101 ++local_max) { 1102 // First match codecs by payload type 1103 if (!requested.Matches(*local_max)) { 1104 continue; 1105 } 1106 1107 out->id = requested.id; 1108 out->name = requested.name; 1109 out->preference = requested.preference; 1110 out->params = requested.params; 1111 out->framerate = talk_base::_min(requested.framerate, local_max->framerate); 1112 out->width = 0; 1113 out->height = 0; 1114 out->params = requested.params; 1115 out->feedback_params = requested.feedback_params; 1116 1117 if (0 == requested.width && 0 == requested.height) { 1118 // Special case with resolution 0. The channel should not send frames. 1119 return true; 1120 } else if (0 == requested.width || 0 == requested.height) { 1121 // 0xn and nx0 are invalid resolutions. 1122 return false; 1123 } 1124 1125 // Pick the best quality that is within their and our bounds and has the 1126 // correct aspect ratio. 1127 for (int j = 0; j < ARRAY_SIZE(kVideoFormats); ++j) { 1128 const VideoFormat format(kVideoFormats[j]); 1129 1130 // Skip any format that is larger than the local or remote maximums, or 1131 // smaller than the current best match 1132 if (format.width > requested.width || format.height > requested.height || 1133 format.width > local_max->width || 1134 (format.width < out->width && format.height < out->height)) { 1135 continue; 1136 } 1137 1138 bool better = false; 1139 1140 // Check any further constraints on this prospective format 1141 if (!out->width || !out->height) { 1142 // If we don't have any matches yet, this is the best so far. 1143 better = true; 1144 } else if (current.width && current.height) { 1145 // current is set so format must match its ratio exactly. 1146 better = 1147 (format.width * current.height == format.height * current.width); 1148 } else { 1149 // Prefer closer aspect ratios i.e 1150 // format.aspect - requested.aspect < out.aspect - requested.aspect 1151 better = abs(format.width * requested.height * out->height - 1152 requested.width * format.height * out->height) < 1153 abs(out->width * format.height * requested.height - 1154 requested.width * format.height * out->height); 1155 } 1156 1157 if (better) { 1158 out->width = format.width; 1159 out->height = format.height; 1160 } 1161 } 1162 if (out->width > 0) { 1163 return true; 1164 } 1165 } 1166 return false; 1167 } 1168 1169 static void ConvertToCricketVideoCodec( 1170 const webrtc::VideoCodec& in_codec, VideoCodec* out_codec) { 1171 out_codec->id = in_codec.plType; 1172 out_codec->name = in_codec.plName; 1173 out_codec->width = in_codec.width; 1174 out_codec->height = in_codec.height; 1175 out_codec->framerate = in_codec.maxFramerate; 1176 out_codec->SetParam(kCodecParamMinBitrate, in_codec.minBitrate); 1177 out_codec->SetParam(kCodecParamMaxBitrate, in_codec.maxBitrate); 1178 if (in_codec.qpMax) { 1179 out_codec->SetParam(kCodecParamMaxQuantization, in_codec.qpMax); 1180 } 1181 } 1182 1183 bool WebRtcVideoEngine::ConvertFromCricketVideoCodec( 1184 const VideoCodec& in_codec, webrtc::VideoCodec* out_codec) { 1185 bool found = false; 1186 int ncodecs = vie_wrapper_->codec()->NumberOfCodecs(); 1187 for (int i = 0; i < ncodecs; ++i) { 1188 if (vie_wrapper_->codec()->GetCodec(i, *out_codec) == 0 && 1189 _stricmp(in_codec.name.c_str(), out_codec->plName) == 0) { 1190 found = true; 1191 break; 1192 } 1193 } 1194 1195 // If not found, check if this is supported by external encoder factory. 1196 if (!found && encoder_factory_) { 1197 const std::vector<WebRtcVideoEncoderFactory::VideoCodec>& codecs = 1198 encoder_factory_->codecs(); 1199 for (size_t i = 0; i < codecs.size(); ++i) { 1200 if (_stricmp(in_codec.name.c_str(), codecs[i].name.c_str()) == 0) { 1201 out_codec->codecType = codecs[i].type; 1202 out_codec->plType = GetExternalVideoPayloadType(static_cast<int>(i)); 1203 talk_base::strcpyn(out_codec->plName, sizeof(out_codec->plName), 1204 codecs[i].name.c_str(), codecs[i].name.length()); 1205 found = true; 1206 break; 1207 } 1208 } 1209 } 1210 1211 if (!found) { 1212 LOG(LS_ERROR) << "invalid codec type"; 1213 return false; 1214 } 1215 1216 if (in_codec.id != 0) 1217 out_codec->plType = in_codec.id; 1218 1219 if (in_codec.width != 0) 1220 out_codec->width = in_codec.width; 1221 1222 if (in_codec.height != 0) 1223 out_codec->height = in_codec.height; 1224 1225 if (in_codec.framerate != 0) 1226 out_codec->maxFramerate = in_codec.framerate; 1227 1228 // Convert bitrate parameters. 1229 int max_bitrate = kMaxVideoBitrate; 1230 int min_bitrate = kMinVideoBitrate; 1231 int start_bitrate = kStartVideoBitrate; 1232 1233 in_codec.GetParam(kCodecParamMinBitrate, &min_bitrate); 1234 in_codec.GetParam(kCodecParamMaxBitrate, &max_bitrate); 1235 1236 if (max_bitrate < min_bitrate) { 1237 return false; 1238 } 1239 start_bitrate = talk_base::_max(start_bitrate, min_bitrate); 1240 start_bitrate = talk_base::_min(start_bitrate, max_bitrate); 1241 1242 out_codec->minBitrate = min_bitrate; 1243 out_codec->startBitrate = start_bitrate; 1244 out_codec->maxBitrate = max_bitrate; 1245 1246 // Convert general codec parameters. 1247 int max_quantization = 0; 1248 if (in_codec.GetParam(kCodecParamMaxQuantization, &max_quantization)) { 1249 if (max_quantization < 0) { 1250 return false; 1251 } 1252 out_codec->qpMax = max_quantization; 1253 } 1254 return true; 1255 } 1256 1257 void WebRtcVideoEngine::RegisterChannel(WebRtcVideoMediaChannel *channel) { 1258 talk_base::CritScope cs(&channels_crit_); 1259 channels_.push_back(channel); 1260 } 1261 1262 void WebRtcVideoEngine::UnregisterChannel(WebRtcVideoMediaChannel *channel) { 1263 talk_base::CritScope cs(&channels_crit_); 1264 channels_.erase(std::remove(channels_.begin(), channels_.end(), channel), 1265 channels_.end()); 1266 } 1267 1268 bool WebRtcVideoEngine::SetVoiceEngine(WebRtcVoiceEngine* voice_engine) { 1269 if (initialized_) { 1270 LOG(LS_WARNING) << "SetVoiceEngine can not be called after Init"; 1271 return false; 1272 } 1273 voice_engine_ = voice_engine; 1274 return true; 1275 } 1276 1277 bool WebRtcVideoEngine::EnableTimedRender() { 1278 if (initialized_) { 1279 LOG(LS_WARNING) << "EnableTimedRender can not be called after Init"; 1280 return false; 1281 } 1282 render_module_.reset(webrtc::VideoRender::CreateVideoRender(0, NULL, 1283 false, webrtc::kRenderExternal)); 1284 return true; 1285 } 1286 1287 void WebRtcVideoEngine::SetTraceFilter(int filter) { 1288 tracing_->SetTraceFilter(filter); 1289 } 1290 1291 // See https://sites.google.com/a/google.com/wavelet/ 1292 // Home/Magic-Flute--RTC-Engine-/Magic-Flute-Command-Line-Parameters 1293 // for all supported command line setttings. 1294 void WebRtcVideoEngine::SetTraceOptions(const std::string& options) { 1295 // Set WebRTC trace file. 1296 std::vector<std::string> opts; 1297 talk_base::tokenize(options, ' ', '"', '"', &opts); 1298 std::vector<std::string>::iterator tracefile = 1299 std::find(opts.begin(), opts.end(), "tracefile"); 1300 if (tracefile != opts.end() && ++tracefile != opts.end()) { 1301 // Write WebRTC debug output (at same loglevel) to file 1302 if (tracing_->SetTraceFile(tracefile->c_str()) == -1) { 1303 LOG_RTCERR1(SetTraceFile, *tracefile); 1304 } 1305 } 1306 } 1307 1308 static void AddDefaultFeedbackParams(VideoCodec* codec) { 1309 const FeedbackParam kFir(kRtcpFbParamCcm, kRtcpFbCcmParamFir); 1310 codec->AddFeedbackParam(kFir); 1311 const FeedbackParam kNack(kRtcpFbParamNack, kParamValueEmpty); 1312 codec->AddFeedbackParam(kNack); 1313 const FeedbackParam kRemb(kRtcpFbParamRemb, kParamValueEmpty); 1314 codec->AddFeedbackParam(kRemb); 1315 } 1316 1317 // Rebuilds the codec list to be only those that are less intensive 1318 // than the specified codec. Prefers internal codec over external with 1319 // higher preference field. 1320 bool WebRtcVideoEngine::RebuildCodecList(const VideoCodec& in_codec) { 1321 if (!FindCodec(in_codec)) 1322 return false; 1323 1324 video_codecs_.clear(); 1325 1326 bool found = false; 1327 std::set<std::string> internal_codec_names; 1328 for (size_t i = 0; i < ARRAY_SIZE(kVideoCodecPrefs); ++i) { 1329 const VideoCodecPref& pref(kVideoCodecPrefs[i]); 1330 if (!found) 1331 found = (in_codec.name == pref.name); 1332 if (found) { 1333 VideoCodec codec(pref.payload_type, pref.name, 1334 in_codec.width, in_codec.height, in_codec.framerate, 1335 static_cast<int>(ARRAY_SIZE(kVideoCodecPrefs) - i)); 1336 if (_stricmp(kVp8PayloadName, codec.name.c_str()) == 0) { 1337 AddDefaultFeedbackParams(&codec); 1338 } 1339 if (pref.associated_payload_type != -1) { 1340 codec.SetParam(kCodecParamAssociatedPayloadType, 1341 pref.associated_payload_type); 1342 } 1343 video_codecs_.push_back(codec); 1344 internal_codec_names.insert(codec.name); 1345 } 1346 } 1347 if (encoder_factory_) { 1348 const std::vector<WebRtcVideoEncoderFactory::VideoCodec>& codecs = 1349 encoder_factory_->codecs(); 1350 for (size_t i = 0; i < codecs.size(); ++i) { 1351 bool is_internal_codec = internal_codec_names.find(codecs[i].name) != 1352 internal_codec_names.end(); 1353 if (!is_internal_codec) { 1354 if (!found) 1355 found = (in_codec.name == codecs[i].name); 1356 VideoCodec codec( 1357 GetExternalVideoPayloadType(static_cast<int>(i)), 1358 codecs[i].name, 1359 codecs[i].max_width, 1360 codecs[i].max_height, 1361 codecs[i].max_fps, 1362 // Use negative preference on external codec to ensure the internal 1363 // codec is preferred. 1364 static_cast<int>(0 - i)); 1365 AddDefaultFeedbackParams(&codec); 1366 video_codecs_.push_back(codec); 1367 } 1368 } 1369 } 1370 ASSERT(found); 1371 return true; 1372 } 1373 1374 // Ignore spammy trace messages, mostly from the stats API when we haven't 1375 // gotten RTCP info yet from the remote side. 1376 bool WebRtcVideoEngine::ShouldIgnoreTrace(const std::string& trace) { 1377 static const char* const kTracesToIgnore[] = { 1378 NULL 1379 }; 1380 for (const char* const* p = kTracesToIgnore; *p; ++p) { 1381 if (trace.find(*p) == 0) { 1382 return true; 1383 } 1384 } 1385 return false; 1386 } 1387 1388 int WebRtcVideoEngine::GetNumOfChannels() { 1389 talk_base::CritScope cs(&channels_crit_); 1390 return static_cast<int>(channels_.size()); 1391 } 1392 1393 void WebRtcVideoEngine::Print(webrtc::TraceLevel level, const char* trace, 1394 int length) { 1395 talk_base::LoggingSeverity sev = talk_base::LS_VERBOSE; 1396 if (level == webrtc::kTraceError || level == webrtc::kTraceCritical) 1397 sev = talk_base::LS_ERROR; 1398 else if (level == webrtc::kTraceWarning) 1399 sev = talk_base::LS_WARNING; 1400 else if (level == webrtc::kTraceStateInfo || level == webrtc::kTraceInfo) 1401 sev = talk_base::LS_INFO; 1402 else if (level == webrtc::kTraceTerseInfo) 1403 sev = talk_base::LS_INFO; 1404 1405 // Skip past boilerplate prefix text 1406 if (length < 72) { 1407 std::string msg(trace, length); 1408 LOG(LS_ERROR) << "Malformed webrtc log message: "; 1409 LOG_V(sev) << msg; 1410 } else { 1411 std::string msg(trace + 71, length - 72); 1412 if (!ShouldIgnoreTrace(msg) && 1413 (!voice_engine_ || !voice_engine_->ShouldIgnoreTrace(msg))) { 1414 LOG_V(sev) << "webrtc: " << msg; 1415 } 1416 } 1417 } 1418 1419 webrtc::VideoDecoder* WebRtcVideoEngine::CreateExternalDecoder( 1420 webrtc::VideoCodecType type) { 1421 if (decoder_factory_ == NULL) { 1422 return NULL; 1423 } 1424 return decoder_factory_->CreateVideoDecoder(type); 1425 } 1426 1427 void WebRtcVideoEngine::DestroyExternalDecoder(webrtc::VideoDecoder* decoder) { 1428 ASSERT(decoder_factory_ != NULL); 1429 if (decoder_factory_ == NULL) 1430 return; 1431 decoder_factory_->DestroyVideoDecoder(decoder); 1432 } 1433 1434 webrtc::VideoEncoder* WebRtcVideoEngine::CreateExternalEncoder( 1435 webrtc::VideoCodecType type) { 1436 if (encoder_factory_ == NULL) { 1437 return NULL; 1438 } 1439 return encoder_factory_->CreateVideoEncoder(type); 1440 } 1441 1442 void WebRtcVideoEngine::DestroyExternalEncoder(webrtc::VideoEncoder* encoder) { 1443 ASSERT(encoder_factory_ != NULL); 1444 if (encoder_factory_ == NULL) 1445 return; 1446 encoder_factory_->DestroyVideoEncoder(encoder); 1447 } 1448 1449 bool WebRtcVideoEngine::IsExternalEncoderCodecType( 1450 webrtc::VideoCodecType type) const { 1451 if (!encoder_factory_) 1452 return false; 1453 const std::vector<WebRtcVideoEncoderFactory::VideoCodec>& codecs = 1454 encoder_factory_->codecs(); 1455 std::vector<WebRtcVideoEncoderFactory::VideoCodec>::const_iterator it; 1456 for (it = codecs.begin(); it != codecs.end(); ++it) { 1457 if (it->type == type) 1458 return true; 1459 } 1460 return false; 1461 } 1462 1463 void WebRtcVideoEngine::SetExternalDecoderFactory( 1464 WebRtcVideoDecoderFactory* decoder_factory) { 1465 decoder_factory_ = decoder_factory; 1466 } 1467 1468 void WebRtcVideoEngine::SetExternalEncoderFactory( 1469 WebRtcVideoEncoderFactory* encoder_factory) { 1470 if (encoder_factory_ == encoder_factory) 1471 return; 1472 1473 if (encoder_factory_) { 1474 encoder_factory_->RemoveObserver(this); 1475 } 1476 encoder_factory_ = encoder_factory; 1477 if (encoder_factory_) { 1478 encoder_factory_->AddObserver(this); 1479 } 1480 1481 // Invoke OnCodecAvailable() here in case the list of codecs is already 1482 // available when the encoder factory is installed. If not the encoder 1483 // factory will invoke the callback later when the codecs become available. 1484 OnCodecsAvailable(); 1485 } 1486 1487 void WebRtcVideoEngine::OnCodecsAvailable() { 1488 // Rebuild codec list while reapplying the current default codec format. 1489 VideoCodec max_codec(kVideoCodecPrefs[0].payload_type, 1490 kVideoCodecPrefs[0].name, 1491 video_codecs_[0].width, 1492 video_codecs_[0].height, 1493 video_codecs_[0].framerate, 1494 0); 1495 if (!RebuildCodecList(max_codec)) { 1496 LOG(LS_ERROR) << "Failed to initialize list of supported codec types"; 1497 } 1498 } 1499 1500 // WebRtcVideoMediaChannel 1501 1502 WebRtcVideoMediaChannel::WebRtcVideoMediaChannel( 1503 WebRtcVideoEngine* engine, 1504 VoiceMediaChannel* channel) 1505 : engine_(engine), 1506 voice_channel_(channel), 1507 vie_channel_(-1), 1508 nack_enabled_(true), 1509 remb_enabled_(false), 1510 render_started_(false), 1511 first_receive_ssrc_(0), 1512 send_rtx_type_(-1), 1513 send_red_type_(-1), 1514 send_fec_type_(-1), 1515 send_min_bitrate_(kMinVideoBitrate), 1516 send_start_bitrate_(kStartVideoBitrate), 1517 send_max_bitrate_(kMaxVideoBitrate), 1518 sending_(false), 1519 ratio_w_(0), 1520 ratio_h_(0) { 1521 engine->RegisterChannel(this); 1522 } 1523 1524 bool WebRtcVideoMediaChannel::Init() { 1525 const uint32 ssrc_key = 0; 1526 return CreateChannel(ssrc_key, MD_SENDRECV, &vie_channel_); 1527 } 1528 1529 WebRtcVideoMediaChannel::~WebRtcVideoMediaChannel() { 1530 const bool send = false; 1531 SetSend(send); 1532 const bool render = false; 1533 SetRender(render); 1534 1535 while (!send_channels_.empty()) { 1536 if (!DeleteSendChannel(send_channels_.begin()->first)) { 1537 LOG(LS_ERROR) << "Unable to delete channel with ssrc key " 1538 << send_channels_.begin()->first; 1539 ASSERT(false); 1540 break; 1541 } 1542 } 1543 1544 // Remove all receive streams and the default channel. 1545 while (!recv_channels_.empty()) { 1546 RemoveRecvStream(recv_channels_.begin()->first); 1547 } 1548 1549 // Unregister the channel from the engine. 1550 engine()->UnregisterChannel(this); 1551 if (worker_thread()) { 1552 worker_thread()->Clear(this); 1553 } 1554 } 1555 1556 bool WebRtcVideoMediaChannel::SetRecvCodecs( 1557 const std::vector<VideoCodec>& codecs) { 1558 receive_codecs_.clear(); 1559 for (std::vector<VideoCodec>::const_iterator iter = codecs.begin(); 1560 iter != codecs.end(); ++iter) { 1561 if (engine()->FindCodec(*iter)) { 1562 webrtc::VideoCodec wcodec; 1563 if (engine()->ConvertFromCricketVideoCodec(*iter, &wcodec)) { 1564 receive_codecs_.push_back(wcodec); 1565 } 1566 } else { 1567 LOG(LS_INFO) << "Unknown codec " << iter->name; 1568 return false; 1569 } 1570 } 1571 1572 for (RecvChannelMap::iterator it = recv_channels_.begin(); 1573 it != recv_channels_.end(); ++it) { 1574 if (!SetReceiveCodecs(it->second)) 1575 return false; 1576 } 1577 return true; 1578 } 1579 1580 bool WebRtcVideoMediaChannel::SetSendCodecs( 1581 const std::vector<VideoCodec>& codecs) { 1582 // Match with local video codec list. 1583 std::vector<webrtc::VideoCodec> send_codecs; 1584 VideoCodec checked_codec; 1585 VideoCodec current; // defaults to 0x0 1586 if (sending_) { 1587 ConvertToCricketVideoCodec(*send_codec_, ¤t); 1588 } 1589 std::map<int, int> primary_rtx_pt_mapping; 1590 for (std::vector<VideoCodec>::const_iterator iter = codecs.begin(); 1591 iter != codecs.end(); ++iter) { 1592 if (_stricmp(iter->name.c_str(), kRedPayloadName) == 0) { 1593 send_red_type_ = iter->id; 1594 } else if (_stricmp(iter->name.c_str(), kFecPayloadName) == 0) { 1595 send_fec_type_ = iter->id; 1596 } else if (_stricmp(iter->name.c_str(), kRtxCodecName) == 0) { 1597 int rtx_type = iter->id; 1598 int rtx_primary_type = -1; 1599 if (iter->GetParam(kCodecParamAssociatedPayloadType, &rtx_primary_type)) { 1600 primary_rtx_pt_mapping[rtx_primary_type] = rtx_type; 1601 } 1602 } else if (engine()->CanSendCodec(*iter, current, &checked_codec)) { 1603 webrtc::VideoCodec wcodec; 1604 if (engine()->ConvertFromCricketVideoCodec(checked_codec, &wcodec)) { 1605 if (send_codecs.empty()) { 1606 nack_enabled_ = IsNackEnabled(checked_codec); 1607 remb_enabled_ = IsRembEnabled(checked_codec); 1608 } 1609 send_codecs.push_back(wcodec); 1610 } 1611 } else { 1612 LOG(LS_WARNING) << "Unknown codec " << iter->name; 1613 } 1614 } 1615 1616 // Fail if we don't have a match. 1617 if (send_codecs.empty()) { 1618 LOG(LS_WARNING) << "No matching codecs available"; 1619 return false; 1620 } 1621 1622 // Recv protection. 1623 for (RecvChannelMap::iterator it = recv_channels_.begin(); 1624 it != recv_channels_.end(); ++it) { 1625 int channel_id = it->second->channel_id(); 1626 if (!SetNackFec(channel_id, send_red_type_, send_fec_type_, 1627 nack_enabled_)) { 1628 return false; 1629 } 1630 if (engine_->vie()->rtp()->SetRembStatus(channel_id, 1631 kNotSending, 1632 remb_enabled_) != 0) { 1633 LOG_RTCERR3(SetRembStatus, channel_id, kNotSending, remb_enabled_); 1634 return false; 1635 } 1636 } 1637 1638 // Send settings. 1639 for (SendChannelMap::iterator iter = send_channels_.begin(); 1640 iter != send_channels_.end(); ++iter) { 1641 int channel_id = iter->second->channel_id(); 1642 if (!SetNackFec(channel_id, send_red_type_, send_fec_type_, 1643 nack_enabled_)) { 1644 return false; 1645 } 1646 if (engine_->vie()->rtp()->SetRembStatus(channel_id, 1647 remb_enabled_, 1648 remb_enabled_) != 0) { 1649 LOG_RTCERR3(SetRembStatus, channel_id, remb_enabled_, remb_enabled_); 1650 return false; 1651 } 1652 } 1653 1654 // Select the first matched codec. 1655 webrtc::VideoCodec& codec(send_codecs[0]); 1656 1657 // Set RTX payload type if primary now active. This value will be used in 1658 // SetSendCodec. 1659 std::map<int, int>::const_iterator rtx_it = 1660 primary_rtx_pt_mapping.find(static_cast<int>(codec.plType)); 1661 if (rtx_it != primary_rtx_pt_mapping.end()) { 1662 send_rtx_type_ = rtx_it->second; 1663 } 1664 1665 if (!SetSendCodec( 1666 codec, codec.minBitrate, codec.startBitrate, codec.maxBitrate)) { 1667 return false; 1668 } 1669 1670 for (SendChannelMap::iterator iter = send_channels_.begin(); 1671 iter != send_channels_.end(); ++iter) { 1672 WebRtcVideoChannelSendInfo* send_channel = iter->second; 1673 send_channel->InitializeAdapterOutputFormat(codec); 1674 } 1675 1676 LogSendCodecChange("SetSendCodecs()"); 1677 1678 return true; 1679 } 1680 1681 bool WebRtcVideoMediaChannel::GetSendCodec(VideoCodec* send_codec) { 1682 if (!send_codec_) { 1683 return false; 1684 } 1685 ConvertToCricketVideoCodec(*send_codec_, send_codec); 1686 return true; 1687 } 1688 1689 bool WebRtcVideoMediaChannel::SetSendStreamFormat(uint32 ssrc, 1690 const VideoFormat& format) { 1691 if (!send_codec_) { 1692 LOG(LS_ERROR) << "The send codec has not been set yet."; 1693 return false; 1694 } 1695 WebRtcVideoChannelSendInfo* send_channel = GetSendChannel(ssrc); 1696 if (!send_channel) { 1697 LOG(LS_ERROR) << "The specified ssrc " << ssrc << " is not in use."; 1698 return false; 1699 } 1700 send_channel->set_video_format(format); 1701 return true; 1702 } 1703 1704 bool WebRtcVideoMediaChannel::SetRender(bool render) { 1705 if (render == render_started_) { 1706 return true; // no action required 1707 } 1708 1709 bool ret = true; 1710 for (RecvChannelMap::iterator it = recv_channels_.begin(); 1711 it != recv_channels_.end(); ++it) { 1712 if (render) { 1713 if (engine()->vie()->render()->StartRender( 1714 it->second->channel_id()) != 0) { 1715 LOG_RTCERR1(StartRender, it->second->channel_id()); 1716 ret = false; 1717 } 1718 } else { 1719 if (engine()->vie()->render()->StopRender( 1720 it->second->channel_id()) != 0) { 1721 LOG_RTCERR1(StopRender, it->second->channel_id()); 1722 ret = false; 1723 } 1724 } 1725 } 1726 if (ret) { 1727 render_started_ = render; 1728 } 1729 1730 return ret; 1731 } 1732 1733 bool WebRtcVideoMediaChannel::SetSend(bool send) { 1734 if (!HasReadySendChannels() && send) { 1735 LOG(LS_ERROR) << "No stream added"; 1736 return false; 1737 } 1738 if (send == sending()) { 1739 return true; // No action required. 1740 } 1741 1742 if (send) { 1743 // We've been asked to start sending. 1744 // SetSendCodecs must have been called already. 1745 if (!send_codec_) { 1746 return false; 1747 } 1748 // Start send now. 1749 if (!StartSend()) { 1750 return false; 1751 } 1752 } else { 1753 // We've been asked to stop sending. 1754 if (!StopSend()) { 1755 return false; 1756 } 1757 } 1758 sending_ = send; 1759 1760 return true; 1761 } 1762 1763 bool WebRtcVideoMediaChannel::AddSendStream(const StreamParams& sp) { 1764 LOG(LS_INFO) << "AddSendStream " << sp.ToString(); 1765 1766 if (!IsOneSsrcStream(sp) && !IsSimulcastStream(sp)) { 1767 LOG(LS_ERROR) << "AddSendStream: bad local stream parameters"; 1768 return false; 1769 } 1770 1771 uint32 ssrc_key; 1772 if (!CreateSendChannelKey(sp.first_ssrc(), &ssrc_key)) { 1773 LOG(LS_ERROR) << "Trying to register duplicate ssrc: " << sp.first_ssrc(); 1774 return false; 1775 } 1776 // If the default channel is already used for sending create a new channel 1777 // otherwise use the default channel for sending. 1778 int channel_id = -1; 1779 if (send_channels_[0]->stream_params() == NULL) { 1780 channel_id = vie_channel_; 1781 } else { 1782 if (!CreateChannel(ssrc_key, MD_SEND, &channel_id)) { 1783 LOG(LS_ERROR) << "AddSendStream: unable to create channel"; 1784 return false; 1785 } 1786 } 1787 WebRtcVideoChannelSendInfo* send_channel = send_channels_[ssrc_key]; 1788 // Set the send (local) SSRC. 1789 // If there are multiple send SSRCs, we can only set the first one here, and 1790 // the rest of the SSRC(s) need to be set after SetSendCodec has been called 1791 // (with a codec requires multiple SSRC(s)). 1792 if (engine()->vie()->rtp()->SetLocalSSRC(channel_id, 1793 sp.first_ssrc()) != 0) { 1794 LOG_RTCERR2(SetLocalSSRC, channel_id, sp.first_ssrc()); 1795 return false; 1796 } 1797 1798 // Set the corresponding RTX SSRC. 1799 if (!SetLocalRtxSsrc(channel_id, sp, sp.first_ssrc(), 0)) { 1800 return false; 1801 } 1802 1803 // Set RTCP CName. 1804 if (engine()->vie()->rtp()->SetRTCPCName(channel_id, 1805 sp.cname.c_str()) != 0) { 1806 LOG_RTCERR2(SetRTCPCName, channel_id, sp.cname.c_str()); 1807 return false; 1808 } 1809 1810 // At this point the channel's local SSRC has been updated. If the channel is 1811 // the default channel make sure that all the receive channels are updated as 1812 // well. Receive channels have to have the same SSRC as the default channel in 1813 // order to send receiver reports with this SSRC. 1814 if (IsDefaultChannel(channel_id)) { 1815 for (RecvChannelMap::const_iterator it = recv_channels_.begin(); 1816 it != recv_channels_.end(); ++it) { 1817 WebRtcVideoChannelRecvInfo* info = it->second; 1818 int channel_id = info->channel_id(); 1819 if (engine()->vie()->rtp()->SetLocalSSRC(channel_id, 1820 sp.first_ssrc()) != 0) { 1821 LOG_RTCERR1(SetLocalSSRC, it->first); 1822 return false; 1823 } 1824 } 1825 } 1826 1827 send_channel->set_stream_params(sp); 1828 1829 // Reset send codec after stream parameters changed. 1830 if (send_codec_) { 1831 if (!SetSendCodec(send_channel, *send_codec_, send_min_bitrate_, 1832 send_start_bitrate_, send_max_bitrate_)) { 1833 return false; 1834 } 1835 LogSendCodecChange("SetSendStreamFormat()"); 1836 } 1837 1838 if (sending_) { 1839 return StartSend(send_channel); 1840 } 1841 return true; 1842 } 1843 1844 bool WebRtcVideoMediaChannel::RemoveSendStream(uint32 ssrc) { 1845 uint32 ssrc_key; 1846 if (!GetSendChannelKey(ssrc, &ssrc_key)) { 1847 LOG(LS_WARNING) << "Try to remove stream with ssrc " << ssrc 1848 << " which doesn't exist."; 1849 return false; 1850 } 1851 WebRtcVideoChannelSendInfo* send_channel = send_channels_[ssrc_key]; 1852 int channel_id = send_channel->channel_id(); 1853 if (IsDefaultChannel(channel_id) && (send_channel->stream_params() == NULL)) { 1854 // Default channel will still exist. However, if stream_params() is NULL 1855 // there is no stream to remove. 1856 return false; 1857 } 1858 if (sending_) { 1859 StopSend(send_channel); 1860 } 1861 1862 const WebRtcVideoChannelSendInfo::EncoderMap& encoder_map = 1863 send_channel->registered_encoders(); 1864 for (WebRtcVideoChannelSendInfo::EncoderMap::const_iterator it = 1865 encoder_map.begin(); it != encoder_map.end(); ++it) { 1866 if (engine()->vie()->ext_codec()->DeRegisterExternalSendCodec( 1867 channel_id, it->first) != 0) { 1868 LOG_RTCERR1(DeregisterEncoderObserver, channel_id); 1869 } 1870 engine()->DestroyExternalEncoder(it->second); 1871 } 1872 send_channel->ClearRegisteredEncoders(); 1873 1874 // The receive channels depend on the default channel, recycle it instead. 1875 if (IsDefaultChannel(channel_id)) { 1876 SetCapturer(GetDefaultChannelSsrc(), NULL); 1877 send_channel->ClearStreamParams(); 1878 } else { 1879 return DeleteSendChannel(ssrc_key); 1880 } 1881 return true; 1882 } 1883 1884 bool WebRtcVideoMediaChannel::AddRecvStream(const StreamParams& sp) { 1885 // TODO(zhurunz) Remove this once BWE works properly across different send 1886 // and receive channels. 1887 // Reuse default channel for recv stream in 1:1 call. 1888 if (!InConferenceMode() && first_receive_ssrc_ == 0) { 1889 LOG(LS_INFO) << "Recv stream " << sp.first_ssrc() 1890 << " reuse default channel #" 1891 << vie_channel_; 1892 first_receive_ssrc_ = sp.first_ssrc(); 1893 if (render_started_) { 1894 if (engine()->vie()->render()->StartRender(vie_channel_) !=0) { 1895 LOG_RTCERR1(StartRender, vie_channel_); 1896 } 1897 } 1898 return true; 1899 } 1900 1901 if (recv_channels_.find(sp.first_ssrc()) != recv_channels_.end() || 1902 first_receive_ssrc_ == sp.first_ssrc()) { 1903 LOG(LS_ERROR) << "Stream already exists"; 1904 return false; 1905 } 1906 1907 // TODO(perkj): Implement recv media from multiple media SSRCs per stream. 1908 // NOTE: We have two SSRCs per stream when RTX is enabled. 1909 if (!IsOneSsrcStream(sp)) { 1910 LOG(LS_ERROR) << "WebRtcVideoMediaChannel supports one primary SSRC per" 1911 << " stream and one FID SSRC per primary SSRC."; 1912 return false; 1913 } 1914 1915 // Create a new channel for receiving video data. 1916 // In order to get the bandwidth estimation work fine for 1917 // receive only channels, we connect all receiving channels 1918 // to our master send channel. 1919 int channel_id = -1; 1920 if (!CreateChannel(sp.first_ssrc(), MD_RECV, &channel_id)) { 1921 return false; 1922 } 1923 1924 // Set the corresponding RTX SSRC. 1925 uint32 rtx_ssrc; 1926 bool has_rtx = sp.GetFidSsrc(sp.first_ssrc(), &rtx_ssrc); 1927 if (has_rtx && engine()->vie()->rtp()->SetRemoteSSRCType( 1928 channel_id, webrtc::kViEStreamTypeRtx, rtx_ssrc) != 0) { 1929 LOG_RTCERR3(SetRemoteSSRCType, channel_id, webrtc::kViEStreamTypeRtx, 1930 rtx_ssrc); 1931 return false; 1932 } 1933 1934 // Get the default renderer. 1935 VideoRenderer* default_renderer = NULL; 1936 if (InConferenceMode()) { 1937 // The recv_channels_ size start out being 1, so if it is two here this 1938 // is the first receive channel created (vie_channel_ is not used for 1939 // receiving in a conference call). This means that the renderer stored 1940 // inside vie_channel_ should be used for the just created channel. 1941 if (recv_channels_.size() == 2 && 1942 recv_channels_.find(0) != recv_channels_.end()) { 1943 GetRenderer(0, &default_renderer); 1944 } 1945 } 1946 1947 // The first recv stream reuses the default renderer (if a default renderer 1948 // has been set). 1949 if (default_renderer) { 1950 SetRenderer(sp.first_ssrc(), default_renderer); 1951 } 1952 1953 LOG(LS_INFO) << "New video stream " << sp.first_ssrc() 1954 << " registered to VideoEngine channel #" 1955 << channel_id << " and connected to channel #" << vie_channel_; 1956 1957 return true; 1958 } 1959 1960 bool WebRtcVideoMediaChannel::RemoveRecvStream(uint32 ssrc) { 1961 RecvChannelMap::iterator it = recv_channels_.find(ssrc); 1962 1963 if (it == recv_channels_.end()) { 1964 // TODO(perkj): Remove this once BWE works properly across different send 1965 // and receive channels. 1966 // The default channel is reused for recv stream in 1:1 call. 1967 if (first_receive_ssrc_ == ssrc) { 1968 first_receive_ssrc_ = 0; 1969 // Need to stop the renderer and remove it since the render window can be 1970 // deleted after this. 1971 if (render_started_) { 1972 if (engine()->vie()->render()->StopRender(vie_channel_) !=0) { 1973 LOG_RTCERR1(StopRender, it->second->channel_id()); 1974 } 1975 } 1976 recv_channels_[0]->SetRenderer(NULL); 1977 return true; 1978 } 1979 return false; 1980 } 1981 WebRtcVideoChannelRecvInfo* info = it->second; 1982 int channel_id = info->channel_id(); 1983 if (engine()->vie()->render()->RemoveRenderer(channel_id) != 0) { 1984 LOG_RTCERR1(RemoveRenderer, channel_id); 1985 } 1986 1987 if (engine()->vie()->network()->DeregisterSendTransport(channel_id) !=0) { 1988 LOG_RTCERR1(DeRegisterSendTransport, channel_id); 1989 } 1990 1991 if (engine()->vie()->codec()->DeregisterDecoderObserver( 1992 channel_id) != 0) { 1993 LOG_RTCERR1(DeregisterDecoderObserver, channel_id); 1994 } 1995 1996 const WebRtcVideoChannelRecvInfo::DecoderMap& decoder_map = 1997 info->registered_decoders(); 1998 for (WebRtcVideoChannelRecvInfo::DecoderMap::const_iterator it = 1999 decoder_map.begin(); it != decoder_map.end(); ++it) { 2000 if (engine()->vie()->ext_codec()->DeRegisterExternalReceiveCodec( 2001 channel_id, it->first) != 0) { 2002 LOG_RTCERR1(DeregisterDecoderObserver, channel_id); 2003 } 2004 engine()->DestroyExternalDecoder(it->second); 2005 } 2006 info->ClearRegisteredDecoders(); 2007 2008 LOG(LS_INFO) << "Removing video stream " << ssrc 2009 << " with VideoEngine channel #" 2010 << channel_id; 2011 bool ret = true; 2012 if (engine()->vie()->base()->DeleteChannel(channel_id) == -1) { 2013 LOG_RTCERR1(DeleteChannel, channel_id); 2014 ret = false; 2015 } 2016 // Delete the WebRtcVideoChannelRecvInfo pointed to by it->second. 2017 delete info; 2018 recv_channels_.erase(it); 2019 return ret; 2020 } 2021 2022 bool WebRtcVideoMediaChannel::StartSend() { 2023 bool success = true; 2024 for (SendChannelMap::iterator iter = send_channels_.begin(); 2025 iter != send_channels_.end(); ++iter) { 2026 WebRtcVideoChannelSendInfo* send_channel = iter->second; 2027 if (!StartSend(send_channel)) { 2028 success = false; 2029 } 2030 } 2031 return success; 2032 } 2033 2034 bool WebRtcVideoMediaChannel::StartSend( 2035 WebRtcVideoChannelSendInfo* send_channel) { 2036 const int channel_id = send_channel->channel_id(); 2037 if (engine()->vie()->base()->StartSend(channel_id) != 0) { 2038 LOG_RTCERR1(StartSend, channel_id); 2039 return false; 2040 } 2041 2042 send_channel->set_sending(true); 2043 return true; 2044 } 2045 2046 bool WebRtcVideoMediaChannel::StopSend() { 2047 bool success = true; 2048 for (SendChannelMap::iterator iter = send_channels_.begin(); 2049 iter != send_channels_.end(); ++iter) { 2050 WebRtcVideoChannelSendInfo* send_channel = iter->second; 2051 if (!StopSend(send_channel)) { 2052 success = false; 2053 } 2054 } 2055 return success; 2056 } 2057 2058 bool WebRtcVideoMediaChannel::StopSend( 2059 WebRtcVideoChannelSendInfo* send_channel) { 2060 const int channel_id = send_channel->channel_id(); 2061 if (engine()->vie()->base()->StopSend(channel_id) != 0) { 2062 LOG_RTCERR1(StopSend, channel_id); 2063 return false; 2064 } 2065 send_channel->set_sending(false); 2066 return true; 2067 } 2068 2069 bool WebRtcVideoMediaChannel::SendIntraFrame() { 2070 bool success = true; 2071 for (SendChannelMap::iterator iter = send_channels_.begin(); 2072 iter != send_channels_.end(); 2073 ++iter) { 2074 WebRtcVideoChannelSendInfo* send_channel = iter->second; 2075 const int channel_id = send_channel->channel_id(); 2076 if (engine()->vie()->codec()->SendKeyFrame(channel_id) != 0) { 2077 LOG_RTCERR1(SendKeyFrame, channel_id); 2078 success = false; 2079 } 2080 } 2081 return success; 2082 } 2083 2084 bool WebRtcVideoMediaChannel::HasReadySendChannels() { 2085 return !send_channels_.empty() && 2086 ((send_channels_.size() > 1) || 2087 (send_channels_[0]->stream_params() != NULL)); 2088 } 2089 2090 bool WebRtcVideoMediaChannel::GetSendChannelKey(uint32 local_ssrc, 2091 uint32* key) { 2092 *key = 0; 2093 // If a send channel is not ready to send it will not have local_ssrc 2094 // registered to it. 2095 if (!HasReadySendChannels()) { 2096 return false; 2097 } 2098 // The default channel is stored with key 0. The key therefore does not match 2099 // the SSRC associated with the default channel. Check if the SSRC provided 2100 // corresponds to the default channel's SSRC. 2101 if (local_ssrc == GetDefaultChannelSsrc()) { 2102 return true; 2103 } 2104 if (send_channels_.find(local_ssrc) == send_channels_.end()) { 2105 for (SendChannelMap::iterator iter = send_channels_.begin(); 2106 iter != send_channels_.end(); ++iter) { 2107 WebRtcVideoChannelSendInfo* send_channel = iter->second; 2108 if (send_channel->has_ssrc(local_ssrc)) { 2109 *key = iter->first; 2110 return true; 2111 } 2112 } 2113 return false; 2114 } 2115 // The key was found in the above std::map::find call. This means that the 2116 // ssrc is the key. 2117 *key = local_ssrc; 2118 return true; 2119 } 2120 2121 WebRtcVideoChannelSendInfo* WebRtcVideoMediaChannel::GetSendChannel( 2122 uint32 local_ssrc) { 2123 uint32 key; 2124 if (!GetSendChannelKey(local_ssrc, &key)) { 2125 return NULL; 2126 } 2127 return send_channels_[key]; 2128 } 2129 2130 bool WebRtcVideoMediaChannel::CreateSendChannelKey(uint32 local_ssrc, 2131 uint32* key) { 2132 if (GetSendChannelKey(local_ssrc, key)) { 2133 // If there is a key corresponding to |local_ssrc|, the SSRC is already in 2134 // use. SSRCs need to be unique in a session and at this point a duplicate 2135 // SSRC has been detected. 2136 return false; 2137 } 2138 if (send_channels_[0]->stream_params() == NULL) { 2139 // key should be 0 here as the default channel should be re-used whenever it 2140 // is not used. 2141 *key = 0; 2142 return true; 2143 } 2144 // SSRC is currently not in use and the default channel is already in use. Use 2145 // the SSRC as key since it is supposed to be unique in a session. 2146 *key = local_ssrc; 2147 return true; 2148 } 2149 2150 int WebRtcVideoMediaChannel::GetSendChannelNum(VideoCapturer* capturer) { 2151 int num = 0; 2152 for (SendChannelMap::iterator iter = send_channels_.begin(); 2153 iter != send_channels_.end(); ++iter) { 2154 WebRtcVideoChannelSendInfo* send_channel = iter->second; 2155 if (send_channel->video_capturer() == capturer) { 2156 ++num; 2157 } 2158 } 2159 return num; 2160 } 2161 2162 uint32 WebRtcVideoMediaChannel::GetDefaultChannelSsrc() { 2163 WebRtcVideoChannelSendInfo* send_channel = send_channels_[0]; 2164 const StreamParams* sp = send_channel->stream_params(); 2165 if (sp == NULL) { 2166 // This happens if no send stream is currently registered. 2167 return 0; 2168 } 2169 return sp->first_ssrc(); 2170 } 2171 2172 bool WebRtcVideoMediaChannel::DeleteSendChannel(uint32 ssrc_key) { 2173 if (send_channels_.find(ssrc_key) == send_channels_.end()) { 2174 return false; 2175 } 2176 WebRtcVideoChannelSendInfo* send_channel = send_channels_[ssrc_key]; 2177 MaybeDisconnectCapturer(send_channel->video_capturer()); 2178 send_channel->set_video_capturer(NULL); 2179 2180 int channel_id = send_channel->channel_id(); 2181 int capture_id = send_channel->capture_id(); 2182 if (engine()->vie()->codec()->DeregisterEncoderObserver( 2183 channel_id) != 0) { 2184 LOG_RTCERR1(DeregisterEncoderObserver, channel_id); 2185 } 2186 2187 // Destroy the external capture interface. 2188 if (engine()->vie()->capture()->DisconnectCaptureDevice( 2189 channel_id) != 0) { 2190 LOG_RTCERR1(DisconnectCaptureDevice, channel_id); 2191 } 2192 if (engine()->vie()->capture()->ReleaseCaptureDevice( 2193 capture_id) != 0) { 2194 LOG_RTCERR1(ReleaseCaptureDevice, capture_id); 2195 } 2196 2197 // The default channel is stored in both |send_channels_| and 2198 // |recv_channels_|. To make sure it is only deleted once from vie let the 2199 // delete call happen when tearing down |recv_channels_| and not here. 2200 if (!IsDefaultChannel(channel_id)) { 2201 engine_->vie()->base()->DeleteChannel(channel_id); 2202 } 2203 delete send_channel; 2204 send_channels_.erase(ssrc_key); 2205 return true; 2206 } 2207 2208 bool WebRtcVideoMediaChannel::RemoveCapturer(uint32 ssrc) { 2209 WebRtcVideoChannelSendInfo* send_channel = GetSendChannel(ssrc); 2210 if (!send_channel) { 2211 return false; 2212 } 2213 VideoCapturer* capturer = send_channel->video_capturer(); 2214 if (capturer == NULL) { 2215 return false; 2216 } 2217 MaybeDisconnectCapturer(capturer); 2218 send_channel->set_video_capturer(NULL); 2219 const int64 timestamp = send_channel->local_stream_info()->time_stamp(); 2220 if (send_codec_) { 2221 QueueBlackFrame(ssrc, timestamp, send_codec_->maxFramerate); 2222 } 2223 return true; 2224 } 2225 2226 bool WebRtcVideoMediaChannel::SetRenderer(uint32 ssrc, 2227 VideoRenderer* renderer) { 2228 if (recv_channels_.find(ssrc) == recv_channels_.end()) { 2229 // TODO(perkj): Remove this once BWE works properly across different send 2230 // and receive channels. 2231 // The default channel is reused for recv stream in 1:1 call. 2232 if (first_receive_ssrc_ == ssrc && 2233 recv_channels_.find(0) != recv_channels_.end()) { 2234 LOG(LS_INFO) << "SetRenderer " << ssrc 2235 << " reuse default channel #" 2236 << vie_channel_; 2237 recv_channels_[0]->SetRenderer(renderer); 2238 return true; 2239 } 2240 return false; 2241 } 2242 2243 recv_channels_[ssrc]->SetRenderer(renderer); 2244 return true; 2245 } 2246 2247 bool WebRtcVideoMediaChannel::GetStats(VideoMediaInfo* info) { 2248 // Get sender statistics and build VideoSenderInfo. 2249 unsigned int total_bitrate_sent = 0; 2250 unsigned int video_bitrate_sent = 0; 2251 unsigned int fec_bitrate_sent = 0; 2252 unsigned int nack_bitrate_sent = 0; 2253 unsigned int estimated_send_bandwidth = 0; 2254 unsigned int target_enc_bitrate = 0; 2255 if (send_codec_) { 2256 for (SendChannelMap::const_iterator iter = send_channels_.begin(); 2257 iter != send_channels_.end(); ++iter) { 2258 WebRtcVideoChannelSendInfo* send_channel = iter->second; 2259 const int channel_id = send_channel->channel_id(); 2260 VideoSenderInfo sinfo; 2261 const StreamParams* send_params = send_channel->stream_params(); 2262 if (send_params == NULL) { 2263 // This should only happen if the default vie channel is not in use. 2264 // This can happen if no streams have ever been added or the stream 2265 // corresponding to the default channel has been removed. Note that 2266 // there may be non-default vie channels in use when this happen so 2267 // asserting send_channels_.size() == 1 is not correct and neither is 2268 // breaking out of the loop. 2269 ASSERT(channel_id == vie_channel_); 2270 continue; 2271 } 2272 unsigned int bytes_sent, packets_sent, bytes_recv, packets_recv; 2273 if (engine_->vie()->rtp()->GetRTPStatistics(channel_id, bytes_sent, 2274 packets_sent, bytes_recv, 2275 packets_recv) != 0) { 2276 LOG_RTCERR1(GetRTPStatistics, vie_channel_); 2277 continue; 2278 } 2279 WebRtcLocalStreamInfo* channel_stream_info = 2280 send_channel->local_stream_info(); 2281 2282 for (size_t i = 0; i < send_params->ssrcs.size(); ++i) { 2283 sinfo.add_ssrc(send_params->ssrcs[i]); 2284 } 2285 sinfo.codec_name = send_codec_->plName; 2286 sinfo.bytes_sent = bytes_sent; 2287 sinfo.packets_sent = packets_sent; 2288 sinfo.packets_cached = -1; 2289 sinfo.packets_lost = -1; 2290 sinfo.fraction_lost = -1; 2291 sinfo.firs_rcvd = -1; 2292 sinfo.nacks_rcvd = -1; 2293 sinfo.rtt_ms = -1; 2294 sinfo.frame_width = static_cast<int>(channel_stream_info->width()); 2295 sinfo.frame_height = static_cast<int>(channel_stream_info->height()); 2296 sinfo.framerate_input = channel_stream_info->framerate(); 2297 sinfo.framerate_sent = send_channel->encoder_observer()->framerate(); 2298 sinfo.nominal_bitrate = send_channel->encoder_observer()->bitrate(); 2299 sinfo.preferred_bitrate = send_max_bitrate_; 2300 sinfo.adapt_reason = send_channel->CurrentAdaptReason(); 2301 sinfo.capture_jitter_ms = -1; 2302 sinfo.avg_encode_ms = -1; 2303 sinfo.encode_usage_percent = -1; 2304 sinfo.capture_queue_delay_ms_per_s = -1; 2305 2306 #ifdef USE_WEBRTC_DEV_BRANCH 2307 int capture_jitter_ms = 0; 2308 int avg_encode_time_ms = 0; 2309 int encode_usage_percent = 0; 2310 int capture_queue_delay_ms_per_s = 0; 2311 if (engine()->vie()->base()->CpuOveruseMeasures( 2312 channel_id, 2313 &capture_jitter_ms, 2314 &avg_encode_time_ms, 2315 &encode_usage_percent, 2316 &capture_queue_delay_ms_per_s) == 0) { 2317 sinfo.capture_jitter_ms = capture_jitter_ms; 2318 sinfo.avg_encode_ms = avg_encode_time_ms; 2319 sinfo.encode_usage_percent = encode_usage_percent; 2320 sinfo.capture_queue_delay_ms_per_s = capture_queue_delay_ms_per_s; 2321 } 2322 #endif 2323 2324 // Get received RTCP statistics for the sender (reported by the remote 2325 // client in a RTCP packet), if available. 2326 // It's not a fatal error if we can't, since RTCP may not have arrived 2327 // yet. 2328 webrtc::RtcpStatistics outgoing_stream_rtcp_stats; 2329 int outgoing_stream_rtt_ms; 2330 2331 if (engine_->vie()->rtp()->GetSendChannelRtcpStatistics( 2332 channel_id, 2333 outgoing_stream_rtcp_stats, 2334 outgoing_stream_rtt_ms) == 0) { 2335 // Convert Q8 to float. 2336 sinfo.packets_lost = outgoing_stream_rtcp_stats.cumulative_lost; 2337 sinfo.fraction_lost = static_cast<float>( 2338 outgoing_stream_rtcp_stats.fraction_lost) / (1 << 8); 2339 sinfo.rtt_ms = outgoing_stream_rtt_ms; 2340 } 2341 info->senders.push_back(sinfo); 2342 2343 unsigned int channel_total_bitrate_sent = 0; 2344 unsigned int channel_video_bitrate_sent = 0; 2345 unsigned int channel_fec_bitrate_sent = 0; 2346 unsigned int channel_nack_bitrate_sent = 0; 2347 if (engine_->vie()->rtp()->GetBandwidthUsage( 2348 channel_id, channel_total_bitrate_sent, channel_video_bitrate_sent, 2349 channel_fec_bitrate_sent, channel_nack_bitrate_sent) == 0) { 2350 total_bitrate_sent += channel_total_bitrate_sent; 2351 video_bitrate_sent += channel_video_bitrate_sent; 2352 fec_bitrate_sent += channel_fec_bitrate_sent; 2353 nack_bitrate_sent += channel_nack_bitrate_sent; 2354 } else { 2355 LOG_RTCERR1(GetBandwidthUsage, channel_id); 2356 } 2357 2358 unsigned int estimated_stream_send_bandwidth = 0; 2359 if (engine_->vie()->rtp()->GetEstimatedSendBandwidth( 2360 channel_id, &estimated_stream_send_bandwidth) == 0) { 2361 estimated_send_bandwidth += estimated_stream_send_bandwidth; 2362 } else { 2363 LOG_RTCERR1(GetEstimatedSendBandwidth, channel_id); 2364 } 2365 unsigned int target_enc_stream_bitrate = 0; 2366 if (engine_->vie()->codec()->GetCodecTargetBitrate( 2367 channel_id, &target_enc_stream_bitrate) == 0) { 2368 target_enc_bitrate += target_enc_stream_bitrate; 2369 } else { 2370 LOG_RTCERR1(GetCodecTargetBitrate, channel_id); 2371 } 2372 } 2373 } else { 2374 LOG(LS_WARNING) << "GetStats: sender information not ready."; 2375 } 2376 2377 // Get the SSRC and stats for each receiver, based on our own calculations. 2378 unsigned int estimated_recv_bandwidth = 0; 2379 for (RecvChannelMap::const_iterator it = recv_channels_.begin(); 2380 it != recv_channels_.end(); ++it) { 2381 // Don't report receive statistics from the default channel if we have 2382 // specified receive channels. 2383 if (it->first == 0 && recv_channels_.size() > 1) 2384 continue; 2385 WebRtcVideoChannelRecvInfo* channel = it->second; 2386 2387 unsigned int ssrc; 2388 // Get receiver statistics and build VideoReceiverInfo, if we have data. 2389 // Skip the default channel (ssrc == 0). 2390 if (engine_->vie()->rtp()->GetRemoteSSRC( 2391 channel->channel_id(), ssrc) != 0 || 2392 ssrc == 0) 2393 continue; 2394 2395 webrtc::StreamDataCounters sent; 2396 webrtc::StreamDataCounters received; 2397 if (engine_->vie()->rtp()->GetRtpStatistics(channel->channel_id(), 2398 sent, received) != 0) { 2399 LOG_RTCERR1(GetRTPStatistics, channel->channel_id()); 2400 return false; 2401 } 2402 VideoReceiverInfo rinfo; 2403 rinfo.add_ssrc(ssrc); 2404 rinfo.bytes_rcvd = received.bytes; 2405 rinfo.packets_rcvd = received.packets; 2406 rinfo.packets_lost = -1; 2407 rinfo.packets_concealed = -1; 2408 rinfo.fraction_lost = -1; // from SentRTCP 2409 rinfo.nacks_sent = -1; 2410 rinfo.frame_width = channel->render_adapter()->width(); 2411 rinfo.frame_height = channel->render_adapter()->height(); 2412 int fps = channel->render_adapter()->framerate(); 2413 rinfo.framerate_decoded = fps; 2414 rinfo.framerate_output = fps; 2415 channel->decoder_observer()->ExportTo(&rinfo); 2416 2417 // Get our locally created statistics of the received RTP stream. 2418 webrtc::RtcpStatistics incoming_stream_rtcp_stats; 2419 int incoming_stream_rtt_ms; 2420 if (engine_->vie()->rtp()->GetReceiveChannelRtcpStatistics( 2421 channel->channel_id(), 2422 incoming_stream_rtcp_stats, 2423 incoming_stream_rtt_ms) == 0) { 2424 // Convert Q8 to float. 2425 rinfo.packets_lost = incoming_stream_rtcp_stats.cumulative_lost; 2426 rinfo.fraction_lost = static_cast<float>( 2427 incoming_stream_rtcp_stats.fraction_lost) / (1 << 8); 2428 } 2429 info->receivers.push_back(rinfo); 2430 2431 unsigned int estimated_recv_stream_bandwidth = 0; 2432 if (engine_->vie()->rtp()->GetEstimatedReceiveBandwidth( 2433 channel->channel_id(), &estimated_recv_stream_bandwidth) == 0) { 2434 estimated_recv_bandwidth += estimated_recv_stream_bandwidth; 2435 } else { 2436 LOG_RTCERR1(GetEstimatedReceiveBandwidth, channel->channel_id()); 2437 } 2438 } 2439 2440 // Build BandwidthEstimationInfo. 2441 // TODO(zhurunz): Add real unittest for this. 2442 BandwidthEstimationInfo bwe; 2443 2444 // Calculations done above per send/receive stream. 2445 bwe.actual_enc_bitrate = video_bitrate_sent; 2446 bwe.transmit_bitrate = total_bitrate_sent; 2447 bwe.retransmit_bitrate = nack_bitrate_sent; 2448 bwe.available_send_bandwidth = estimated_send_bandwidth; 2449 bwe.available_recv_bandwidth = estimated_recv_bandwidth; 2450 bwe.target_enc_bitrate = target_enc_bitrate; 2451 2452 info->bw_estimations.push_back(bwe); 2453 2454 return true; 2455 } 2456 2457 bool WebRtcVideoMediaChannel::SetCapturer(uint32 ssrc, 2458 VideoCapturer* capturer) { 2459 ASSERT(ssrc != 0); 2460 if (!capturer) { 2461 return RemoveCapturer(ssrc); 2462 } 2463 WebRtcVideoChannelSendInfo* send_channel = GetSendChannel(ssrc); 2464 if (!send_channel) { 2465 return false; 2466 } 2467 VideoCapturer* old_capturer = send_channel->video_capturer(); 2468 MaybeDisconnectCapturer(old_capturer); 2469 2470 send_channel->set_video_capturer(capturer); 2471 MaybeConnectCapturer(capturer); 2472 if (!capturer->IsScreencast() && ratio_w_ != 0 && ratio_h_ != 0) { 2473 capturer->UpdateAspectRatio(ratio_w_, ratio_h_); 2474 } 2475 const int64 timestamp = send_channel->local_stream_info()->time_stamp(); 2476 if (send_codec_) { 2477 QueueBlackFrame(ssrc, timestamp, send_codec_->maxFramerate); 2478 } 2479 return true; 2480 } 2481 2482 bool WebRtcVideoMediaChannel::RequestIntraFrame() { 2483 // There is no API exposed to application to request a key frame 2484 // ViE does this internally when there are errors from decoder 2485 return false; 2486 } 2487 2488 void WebRtcVideoMediaChannel::OnPacketReceived( 2489 talk_base::Buffer* packet, const talk_base::PacketTime& packet_time) { 2490 // Pick which channel to send this packet to. If this packet doesn't match 2491 // any multiplexed streams, just send it to the default channel. Otherwise, 2492 // send it to the specific decoder instance for that stream. 2493 uint32 ssrc = 0; 2494 if (!GetRtpSsrc(packet->data(), packet->length(), &ssrc)) 2495 return; 2496 int which_channel = GetRecvChannelNum(ssrc); 2497 if (which_channel == -1) { 2498 which_channel = video_channel(); 2499 } 2500 2501 engine()->vie()->network()->ReceivedRTPPacket( 2502 which_channel, 2503 packet->data(), 2504 #ifdef USE_WEBRTC_DEV_BRANCH 2505 static_cast<int>(packet->length()), 2506 webrtc::PacketTime(packet_time.timestamp, packet_time.not_before)); 2507 #else 2508 static_cast<int>(packet->length())); 2509 #endif 2510 } 2511 2512 void WebRtcVideoMediaChannel::OnRtcpReceived( 2513 talk_base::Buffer* packet, const talk_base::PacketTime& packet_time) { 2514 // Sending channels need all RTCP packets with feedback information. 2515 // Even sender reports can contain attached report blocks. 2516 // Receiving channels need sender reports in order to create 2517 // correct receiver reports. 2518 2519 uint32 ssrc = 0; 2520 if (!GetRtcpSsrc(packet->data(), packet->length(), &ssrc)) { 2521 LOG(LS_WARNING) << "Failed to parse SSRC from received RTCP packet"; 2522 return; 2523 } 2524 int type = 0; 2525 if (!GetRtcpType(packet->data(), packet->length(), &type)) { 2526 LOG(LS_WARNING) << "Failed to parse type from received RTCP packet"; 2527 return; 2528 } 2529 2530 // If it is a sender report, find the channel that is listening. 2531 if (type == kRtcpTypeSR) { 2532 int which_channel = GetRecvChannelNum(ssrc); 2533 if (which_channel != -1 && !IsDefaultChannel(which_channel)) { 2534 engine_->vie()->network()->ReceivedRTCPPacket( 2535 which_channel, 2536 packet->data(), 2537 static_cast<int>(packet->length())); 2538 } 2539 } 2540 // SR may continue RR and any RR entry may correspond to any one of the send 2541 // channels. So all RTCP packets must be forwarded all send channels. ViE 2542 // will filter out RR internally. 2543 for (SendChannelMap::iterator iter = send_channels_.begin(); 2544 iter != send_channels_.end(); ++iter) { 2545 WebRtcVideoChannelSendInfo* send_channel = iter->second; 2546 int channel_id = send_channel->channel_id(); 2547 engine_->vie()->network()->ReceivedRTCPPacket( 2548 channel_id, 2549 packet->data(), 2550 static_cast<int>(packet->length())); 2551 } 2552 } 2553 2554 void WebRtcVideoMediaChannel::OnReadyToSend(bool ready) { 2555 SetNetworkTransmissionState(ready); 2556 } 2557 2558 bool WebRtcVideoMediaChannel::MuteStream(uint32 ssrc, bool muted) { 2559 WebRtcVideoChannelSendInfo* send_channel = GetSendChannel(ssrc); 2560 if (!send_channel) { 2561 LOG(LS_ERROR) << "The specified ssrc " << ssrc << " is not in use."; 2562 return false; 2563 } 2564 send_channel->set_muted(muted); 2565 return true; 2566 } 2567 2568 bool WebRtcVideoMediaChannel::SetRecvRtpHeaderExtensions( 2569 const std::vector<RtpHeaderExtension>& extensions) { 2570 if (receive_extensions_ == extensions) { 2571 return true; 2572 } 2573 receive_extensions_ = extensions; 2574 2575 const RtpHeaderExtension* offset_extension = 2576 FindHeaderExtension(extensions, kRtpTimestampOffsetHeaderExtension); 2577 const RtpHeaderExtension* send_time_extension = 2578 FindHeaderExtension(extensions, kRtpAbsoluteSendTimeHeaderExtension); 2579 2580 // Loop through all receive channels and enable/disable the extensions. 2581 for (RecvChannelMap::iterator channel_it = recv_channels_.begin(); 2582 channel_it != recv_channels_.end(); ++channel_it) { 2583 int channel_id = channel_it->second->channel_id(); 2584 if (!SetHeaderExtension( 2585 &webrtc::ViERTP_RTCP::SetReceiveTimestampOffsetStatus, channel_id, 2586 offset_extension)) { 2587 return false; 2588 } 2589 if (!SetHeaderExtension( 2590 &webrtc::ViERTP_RTCP::SetReceiveAbsoluteSendTimeStatus, channel_id, 2591 send_time_extension)) { 2592 return false; 2593 } 2594 } 2595 return true; 2596 } 2597 2598 bool WebRtcVideoMediaChannel::SetSendRtpHeaderExtensions( 2599 const std::vector<RtpHeaderExtension>& extensions) { 2600 send_extensions_ = extensions; 2601 2602 const RtpHeaderExtension* offset_extension = 2603 FindHeaderExtension(extensions, kRtpTimestampOffsetHeaderExtension); 2604 const RtpHeaderExtension* send_time_extension = 2605 FindHeaderExtension(extensions, kRtpAbsoluteSendTimeHeaderExtension); 2606 2607 // Loop through all send channels and enable/disable the extensions. 2608 for (SendChannelMap::iterator channel_it = send_channels_.begin(); 2609 channel_it != send_channels_.end(); ++channel_it) { 2610 int channel_id = channel_it->second->channel_id(); 2611 if (!SetHeaderExtension( 2612 &webrtc::ViERTP_RTCP::SetSendTimestampOffsetStatus, channel_id, 2613 offset_extension)) { 2614 return false; 2615 } 2616 if (!SetHeaderExtension( 2617 &webrtc::ViERTP_RTCP::SetSendAbsoluteSendTimeStatus, channel_id, 2618 send_time_extension)) { 2619 return false; 2620 } 2621 } 2622 return true; 2623 } 2624 2625 bool WebRtcVideoMediaChannel::SetSendBandwidth(bool autobw, int bps) { 2626 LOG(LS_INFO) << "WebRtcVideoMediaChanne::SetSendBandwidth"; 2627 2628 if (InConferenceMode()) { 2629 LOG(LS_INFO) << "Conference mode ignores SetSendBandWidth"; 2630 return true; 2631 } 2632 2633 if (!send_codec_) { 2634 LOG(LS_INFO) << "The send codec has not been set up yet"; 2635 return true; 2636 } 2637 2638 int min_bitrate; 2639 int start_bitrate; 2640 int max_bitrate; 2641 if (autobw) { 2642 // Use the default values for min bitrate. 2643 min_bitrate = send_min_bitrate_; 2644 // Use the default value or the bps for the max 2645 max_bitrate = (bps <= 0) ? send_max_bitrate_ : (bps / 1000); 2646 // Maximum start bitrate can be kStartVideoBitrate. 2647 start_bitrate = talk_base::_min(kStartVideoBitrate, max_bitrate); 2648 } else { 2649 // Use the default start or the bps as the target bitrate. 2650 int target_bitrate = (bps <= 0) ? kStartVideoBitrate : (bps / 1000); 2651 min_bitrate = target_bitrate; 2652 start_bitrate = target_bitrate; 2653 max_bitrate = target_bitrate; 2654 } 2655 2656 if (!SetSendCodec(*send_codec_, min_bitrate, start_bitrate, max_bitrate)) { 2657 return false; 2658 } 2659 LogSendCodecChange("SetSendBandwidth()"); 2660 2661 return true; 2662 } 2663 2664 bool WebRtcVideoMediaChannel::SetOptions(const VideoOptions &options) { 2665 // Always accept options that are unchanged. 2666 if (options_ == options) { 2667 return true; 2668 } 2669 2670 // Trigger SetSendCodec to set correct noise reduction state if the option has 2671 // changed. 2672 bool denoiser_changed = options.video_noise_reduction.IsSet() && 2673 (options_.video_noise_reduction != options.video_noise_reduction); 2674 2675 bool leaky_bucket_changed = options.video_leaky_bucket.IsSet() && 2676 (options_.video_leaky_bucket != options.video_leaky_bucket); 2677 2678 bool buffer_latency_changed = options.buffered_mode_latency.IsSet() && 2679 (options_.buffered_mode_latency != options.buffered_mode_latency); 2680 2681 bool cpu_overuse_detection_changed = options.cpu_overuse_detection.IsSet() && 2682 (options_.cpu_overuse_detection != options.cpu_overuse_detection); 2683 2684 bool dscp_option_changed = (options_.dscp != options.dscp); 2685 2686 bool suspend_below_min_bitrate_changed = 2687 options.suspend_below_min_bitrate.IsSet() && 2688 (options_.suspend_below_min_bitrate != options.suspend_below_min_bitrate); 2689 2690 bool conference_mode_turned_off = false; 2691 if (options_.conference_mode.IsSet() && options.conference_mode.IsSet() && 2692 options_.conference_mode.GetWithDefaultIfUnset(false) && 2693 !options.conference_mode.GetWithDefaultIfUnset(false)) { 2694 conference_mode_turned_off = true; 2695 } 2696 2697 // Save the options, to be interpreted where appropriate. 2698 // Use options_.SetAll() instead of assignment so that unset value in options 2699 // will not overwrite the previous option value. 2700 options_.SetAll(options); 2701 2702 // Set CPU options for all send channels. 2703 for (SendChannelMap::iterator iter = send_channels_.begin(); 2704 iter != send_channels_.end(); ++iter) { 2705 WebRtcVideoChannelSendInfo* send_channel = iter->second; 2706 send_channel->ApplyCpuOptions(options_); 2707 } 2708 2709 // Adjust send codec bitrate if needed. 2710 int conf_max_bitrate = kDefaultConferenceModeMaxVideoBitrate; 2711 2712 // Save altered min_bitrate level and apply if necessary. 2713 bool adjusted_min_bitrate = false; 2714 if (options.lower_min_bitrate.IsSet()) { 2715 bool lower; 2716 options.lower_min_bitrate.Get(&lower); 2717 2718 int new_send_min_bitrate = lower ? kLowerMinBitrate : kMinVideoBitrate; 2719 adjusted_min_bitrate = (new_send_min_bitrate != send_min_bitrate_); 2720 send_min_bitrate_ = new_send_min_bitrate; 2721 } 2722 2723 int expected_bitrate = send_max_bitrate_; 2724 if (InConferenceMode()) { 2725 expected_bitrate = conf_max_bitrate; 2726 } else if (conference_mode_turned_off) { 2727 // This is a special case for turning conference mode off. 2728 // Max bitrate should go back to the default maximum value instead 2729 // of the current maximum. 2730 expected_bitrate = kMaxVideoBitrate; 2731 } 2732 2733 if (send_codec_ && 2734 (send_max_bitrate_ != expected_bitrate || denoiser_changed || 2735 adjusted_min_bitrate)) { 2736 // On success, SetSendCodec() will reset send_max_bitrate_ to 2737 // expected_bitrate. 2738 if (!SetSendCodec(*send_codec_, 2739 send_min_bitrate_, 2740 send_start_bitrate_, 2741 expected_bitrate)) { 2742 return false; 2743 } 2744 LogSendCodecChange("SetOptions()"); 2745 } 2746 if (leaky_bucket_changed) { 2747 bool enable_leaky_bucket = 2748 options_.video_leaky_bucket.GetWithDefaultIfUnset(false); 2749 for (SendChannelMap::iterator it = send_channels_.begin(); 2750 it != send_channels_.end(); ++it) { 2751 if (engine()->vie()->rtp()->SetTransmissionSmoothingStatus( 2752 it->second->channel_id(), enable_leaky_bucket) != 0) { 2753 LOG_RTCERR2(SetTransmissionSmoothingStatus, it->second->channel_id(), 2754 enable_leaky_bucket); 2755 } 2756 } 2757 } 2758 if (buffer_latency_changed) { 2759 int buffer_latency = 2760 options_.buffered_mode_latency.GetWithDefaultIfUnset( 2761 cricket::kBufferedModeDisabled); 2762 for (SendChannelMap::iterator it = send_channels_.begin(); 2763 it != send_channels_.end(); ++it) { 2764 if (engine()->vie()->rtp()->SetSenderBufferingMode( 2765 it->second->channel_id(), buffer_latency) != 0) { 2766 LOG_RTCERR2(SetSenderBufferingMode, it->second->channel_id(), 2767 buffer_latency); 2768 } 2769 } 2770 for (RecvChannelMap::iterator it = recv_channels_.begin(); 2771 it != recv_channels_.end(); ++it) { 2772 if (engine()->vie()->rtp()->SetReceiverBufferingMode( 2773 it->second->channel_id(), buffer_latency) != 0) { 2774 LOG_RTCERR2(SetReceiverBufferingMode, it->second->channel_id(), 2775 buffer_latency); 2776 } 2777 } 2778 } 2779 if (cpu_overuse_detection_changed) { 2780 bool cpu_overuse_detection = 2781 options_.cpu_overuse_detection.GetWithDefaultIfUnset(false); 2782 for (SendChannelMap::iterator iter = send_channels_.begin(); 2783 iter != send_channels_.end(); ++iter) { 2784 WebRtcVideoChannelSendInfo* send_channel = iter->second; 2785 send_channel->SetCpuOveruseDetection(cpu_overuse_detection); 2786 } 2787 } 2788 if (dscp_option_changed) { 2789 talk_base::DiffServCodePoint dscp = talk_base::DSCP_DEFAULT; 2790 if (options.dscp.GetWithDefaultIfUnset(false)) 2791 dscp = kVideoDscpValue; 2792 if (MediaChannel::SetDscp(dscp) != 0) { 2793 LOG(LS_WARNING) << "Failed to set DSCP settings for video channel"; 2794 } 2795 } 2796 if (suspend_below_min_bitrate_changed) { 2797 if (options_.suspend_below_min_bitrate.GetWithDefaultIfUnset(false)) { 2798 for (SendChannelMap::iterator it = send_channels_.begin(); 2799 it != send_channels_.end(); ++it) { 2800 engine()->vie()->codec()->SuspendBelowMinBitrate( 2801 it->second->channel_id()); 2802 } 2803 } else { 2804 LOG(LS_WARNING) << "Cannot disable video suspension once it is enabled"; 2805 } 2806 } 2807 return true; 2808 } 2809 2810 void WebRtcVideoMediaChannel::SetInterface(NetworkInterface* iface) { 2811 MediaChannel::SetInterface(iface); 2812 // Set the RTP recv/send buffer to a bigger size 2813 MediaChannel::SetOption(NetworkInterface::ST_RTP, 2814 talk_base::Socket::OPT_RCVBUF, 2815 kVideoRtpBufferSize); 2816 2817 // TODO(sriniv): Remove or re-enable this. 2818 // As part of b/8030474, send-buffer is size now controlled through 2819 // portallocator flags. 2820 // network_interface_->SetOption(NetworkInterface::ST_RTP, 2821 // talk_base::Socket::OPT_SNDBUF, 2822 // kVideoRtpBufferSize); 2823 } 2824 2825 void WebRtcVideoMediaChannel::UpdateAspectRatio(int ratio_w, int ratio_h) { 2826 ASSERT(ratio_w != 0); 2827 ASSERT(ratio_h != 0); 2828 ratio_w_ = ratio_w; 2829 ratio_h_ = ratio_h; 2830 // For now assume that all streams want the same aspect ratio. 2831 // TODO(hellner): remove the need for this assumption. 2832 for (SendChannelMap::iterator iter = send_channels_.begin(); 2833 iter != send_channels_.end(); ++iter) { 2834 WebRtcVideoChannelSendInfo* send_channel = iter->second; 2835 VideoCapturer* capturer = send_channel->video_capturer(); 2836 if (capturer) { 2837 capturer->UpdateAspectRatio(ratio_w, ratio_h); 2838 } 2839 } 2840 } 2841 2842 bool WebRtcVideoMediaChannel::GetRenderer(uint32 ssrc, 2843 VideoRenderer** renderer) { 2844 RecvChannelMap::const_iterator it = recv_channels_.find(ssrc); 2845 if (it == recv_channels_.end()) { 2846 if (first_receive_ssrc_ == ssrc && 2847 recv_channels_.find(0) != recv_channels_.end()) { 2848 LOG(LS_INFO) << " GetRenderer " << ssrc 2849 << " reuse default renderer #" 2850 << vie_channel_; 2851 *renderer = recv_channels_[0]->render_adapter()->renderer(); 2852 return true; 2853 } 2854 return false; 2855 } 2856 2857 *renderer = it->second->render_adapter()->renderer(); 2858 return true; 2859 } 2860 2861 void WebRtcVideoMediaChannel::SendFrame(VideoCapturer* capturer, 2862 const VideoFrame* frame) { 2863 // If the |capturer| is registered to any send channel, then send the frame 2864 // to those send channels. 2865 bool capturer_is_channel_owned = false; 2866 for (SendChannelMap::iterator iter = send_channels_.begin(); 2867 iter != send_channels_.end(); ++iter) { 2868 WebRtcVideoChannelSendInfo* send_channel = iter->second; 2869 if (send_channel->video_capturer() == capturer) { 2870 SendFrame(send_channel, frame, capturer->IsScreencast()); 2871 capturer_is_channel_owned = true; 2872 } 2873 } 2874 if (capturer_is_channel_owned) { 2875 return; 2876 } 2877 2878 // TODO(hellner): Remove below for loop once the captured frame no longer 2879 // come from the engine, i.e. the engine no longer owns a capturer. 2880 for (SendChannelMap::iterator iter = send_channels_.begin(); 2881 iter != send_channels_.end(); ++iter) { 2882 WebRtcVideoChannelSendInfo* send_channel = iter->second; 2883 if (send_channel->video_capturer() == NULL) { 2884 SendFrame(send_channel, frame, capturer->IsScreencast()); 2885 } 2886 } 2887 } 2888 2889 bool WebRtcVideoMediaChannel::SendFrame( 2890 WebRtcVideoChannelSendInfo* send_channel, 2891 const VideoFrame* frame, 2892 bool is_screencast) { 2893 if (!send_channel) { 2894 return false; 2895 } 2896 if (!send_codec_) { 2897 // Send codec has not been set. No reason to process the frame any further. 2898 return false; 2899 } 2900 const VideoFormat& video_format = send_channel->video_format(); 2901 // If the frame should be dropped. 2902 const bool video_format_set = video_format != cricket::VideoFormat(); 2903 if (video_format_set && 2904 (video_format.width == 0 && video_format.height == 0)) { 2905 return true; 2906 } 2907 2908 // Checks if we need to reset vie send codec. 2909 if (!MaybeResetVieSendCodec(send_channel, 2910 static_cast<int>(frame->GetWidth()), 2911 static_cast<int>(frame->GetHeight()), 2912 is_screencast, NULL)) { 2913 LOG(LS_ERROR) << "MaybeResetVieSendCodec failed with " 2914 << frame->GetWidth() << "x" << frame->GetHeight(); 2915 return false; 2916 } 2917 const VideoFrame* frame_out = frame; 2918 talk_base::scoped_ptr<VideoFrame> processed_frame; 2919 // Disable muting for screencast. 2920 const bool mute = (send_channel->muted() && !is_screencast); 2921 send_channel->ProcessFrame(*frame_out, mute, processed_frame.use()); 2922 if (processed_frame) { 2923 frame_out = processed_frame.get(); 2924 } 2925 2926 webrtc::ViEVideoFrameI420 frame_i420; 2927 // TODO(ronghuawu): Update the webrtc::ViEVideoFrameI420 2928 // to use const unsigned char* 2929 frame_i420.y_plane = const_cast<unsigned char*>(frame_out->GetYPlane()); 2930 frame_i420.u_plane = const_cast<unsigned char*>(frame_out->GetUPlane()); 2931 frame_i420.v_plane = const_cast<unsigned char*>(frame_out->GetVPlane()); 2932 frame_i420.y_pitch = frame_out->GetYPitch(); 2933 frame_i420.u_pitch = frame_out->GetUPitch(); 2934 frame_i420.v_pitch = frame_out->GetVPitch(); 2935 frame_i420.width = static_cast<uint16>(frame_out->GetWidth()); 2936 frame_i420.height = static_cast<uint16>(frame_out->GetHeight()); 2937 2938 int64 timestamp_ntp_ms = 0; 2939 // TODO(justinlin): Reenable after Windows issues with clock drift are fixed. 2940 // Currently reverted to old behavior of discarding capture timestamp. 2941 #if 0 2942 // If the frame timestamp is 0, we will use the deliver time. 2943 const int64 frame_timestamp = frame->GetTimeStamp(); 2944 if (frame_timestamp != 0) { 2945 if (abs(time(NULL) - frame_timestamp / talk_base::kNumNanosecsPerSec) > 2946 kTimestampDeltaInSecondsForWarning) { 2947 LOG(LS_WARNING) << "Frame timestamp differs by more than " 2948 << kTimestampDeltaInSecondsForWarning << " seconds from " 2949 << "current Unix timestamp."; 2950 } 2951 2952 timestamp_ntp_ms = 2953 talk_base::UnixTimestampNanosecsToNtpMillisecs(frame_timestamp); 2954 } 2955 #endif 2956 2957 return send_channel->external_capture()->IncomingFrameI420( 2958 frame_i420, timestamp_ntp_ms) == 0; 2959 } 2960 2961 bool WebRtcVideoMediaChannel::CreateChannel(uint32 ssrc_key, 2962 MediaDirection direction, 2963 int* channel_id) { 2964 // There are 3 types of channels. Sending only, receiving only and 2965 // sending and receiving. The sending and receiving channel is the 2966 // default channel and there is only one. All other channels that are created 2967 // are associated with the default channel which must exist. The default 2968 // channel id is stored in |vie_channel_|. All channels need to know about 2969 // the default channel to properly handle remb which is why there are 2970 // different ViE create channel calls. 2971 // For this channel the local and remote ssrc key is 0. However, it may 2972 // have a non-zero local and/or remote ssrc depending on if it is currently 2973 // sending and/or receiving. 2974 if ((vie_channel_ == -1 || direction == MD_SENDRECV) && 2975 (!send_channels_.empty() || !recv_channels_.empty())) { 2976 ASSERT(false); 2977 return false; 2978 } 2979 2980 *channel_id = -1; 2981 if (direction == MD_RECV) { 2982 // All rec channels are associated with the default channel |vie_channel_| 2983 if (engine_->vie()->base()->CreateReceiveChannel(*channel_id, 2984 vie_channel_) != 0) { 2985 LOG_RTCERR2(CreateReceiveChannel, *channel_id, vie_channel_); 2986 return false; 2987 } 2988 } else if (direction == MD_SEND) { 2989 if (engine_->vie()->base()->CreateChannel(*channel_id, 2990 vie_channel_) != 0) { 2991 LOG_RTCERR2(CreateChannel, *channel_id, vie_channel_); 2992 return false; 2993 } 2994 } else { 2995 ASSERT(direction == MD_SENDRECV); 2996 if (engine_->vie()->base()->CreateChannel(*channel_id) != 0) { 2997 LOG_RTCERR1(CreateChannel, *channel_id); 2998 return false; 2999 } 3000 } 3001 if (!ConfigureChannel(*channel_id, direction, ssrc_key)) { 3002 engine_->vie()->base()->DeleteChannel(*channel_id); 3003 *channel_id = -1; 3004 return false; 3005 } 3006 3007 return true; 3008 } 3009 3010 bool WebRtcVideoMediaChannel::ConfigureChannel(int channel_id, 3011 MediaDirection direction, 3012 uint32 ssrc_key) { 3013 const bool receiving = (direction == MD_RECV) || (direction == MD_SENDRECV); 3014 const bool sending = (direction == MD_SEND) || (direction == MD_SENDRECV); 3015 // Register external transport. 3016 if (engine_->vie()->network()->RegisterSendTransport( 3017 channel_id, *this) != 0) { 3018 LOG_RTCERR1(RegisterSendTransport, channel_id); 3019 return false; 3020 } 3021 3022 // Set MTU. 3023 if (engine_->vie()->network()->SetMTU(channel_id, kVideoMtu) != 0) { 3024 LOG_RTCERR2(SetMTU, channel_id, kVideoMtu); 3025 return false; 3026 } 3027 // Turn on RTCP and loss feedback reporting. 3028 if (engine()->vie()->rtp()->SetRTCPStatus( 3029 channel_id, webrtc::kRtcpCompound_RFC4585) != 0) { 3030 LOG_RTCERR2(SetRTCPStatus, channel_id, webrtc::kRtcpCompound_RFC4585); 3031 return false; 3032 } 3033 // Enable pli as key frame request method. 3034 if (engine_->vie()->rtp()->SetKeyFrameRequestMethod( 3035 channel_id, webrtc::kViEKeyFrameRequestPliRtcp) != 0) { 3036 LOG_RTCERR2(SetKeyFrameRequestMethod, 3037 channel_id, webrtc::kViEKeyFrameRequestPliRtcp); 3038 return false; 3039 } 3040 if (!SetNackFec(channel_id, send_red_type_, send_fec_type_, nack_enabled_)) { 3041 // Logged in SetNackFec. Don't spam the logs. 3042 return false; 3043 } 3044 // Note that receiving must always be configured before sending to ensure 3045 // that send and receive channel is configured correctly (ConfigureReceiving 3046 // assumes no sending). 3047 if (receiving) { 3048 if (!ConfigureReceiving(channel_id, ssrc_key)) { 3049 return false; 3050 } 3051 } 3052 if (sending) { 3053 if (!ConfigureSending(channel_id, ssrc_key)) { 3054 return false; 3055 } 3056 } 3057 3058 return true; 3059 } 3060 3061 bool WebRtcVideoMediaChannel::ConfigureReceiving(int channel_id, 3062 uint32 remote_ssrc_key) { 3063 // Make sure that an SSRC/key isn't registered more than once. 3064 if (recv_channels_.find(remote_ssrc_key) != recv_channels_.end()) { 3065 return false; 3066 } 3067 // Connect the voice channel, if there is one. 3068 // TODO(perkj): The A/V is synched by the receiving channel. So we need to 3069 // know the SSRC of the remote audio channel in order to fetch the correct 3070 // webrtc VoiceEngine channel. For now- only sync the default channel used 3071 // in 1-1 calls. 3072 if (remote_ssrc_key == 0 && voice_channel_) { 3073 WebRtcVoiceMediaChannel* voice_channel = 3074 static_cast<WebRtcVoiceMediaChannel*>(voice_channel_); 3075 if (engine_->vie()->base()->ConnectAudioChannel( 3076 vie_channel_, voice_channel->voe_channel()) != 0) { 3077 LOG_RTCERR2(ConnectAudioChannel, channel_id, 3078 voice_channel->voe_channel()); 3079 LOG(LS_WARNING) << "A/V not synchronized"; 3080 // Not a fatal error. 3081 } 3082 } 3083 3084 talk_base::scoped_ptr<WebRtcVideoChannelRecvInfo> channel_info( 3085 new WebRtcVideoChannelRecvInfo(channel_id)); 3086 3087 // Install a render adapter. 3088 if (engine_->vie()->render()->AddRenderer(channel_id, 3089 webrtc::kVideoI420, channel_info->render_adapter()) != 0) { 3090 LOG_RTCERR3(AddRenderer, channel_id, webrtc::kVideoI420, 3091 channel_info->render_adapter()); 3092 return false; 3093 } 3094 3095 3096 if (engine_->vie()->rtp()->SetRembStatus(channel_id, 3097 kNotSending, 3098 remb_enabled_) != 0) { 3099 LOG_RTCERR3(SetRembStatus, channel_id, kNotSending, remb_enabled_); 3100 return false; 3101 } 3102 3103 if (!SetHeaderExtension(&webrtc::ViERTP_RTCP::SetReceiveTimestampOffsetStatus, 3104 channel_id, receive_extensions_, kRtpTimestampOffsetHeaderExtension)) { 3105 return false; 3106 } 3107 3108 if (!SetHeaderExtension( 3109 &webrtc::ViERTP_RTCP::SetReceiveAbsoluteSendTimeStatus, channel_id, 3110 receive_extensions_, kRtpAbsoluteSendTimeHeaderExtension)) { 3111 return false; 3112 } 3113 3114 if (remote_ssrc_key != 0) { 3115 // Use the same SSRC as our default channel 3116 // (so the RTCP reports are correct). 3117 unsigned int send_ssrc = 0; 3118 webrtc::ViERTP_RTCP* rtp = engine()->vie()->rtp(); 3119 if (rtp->GetLocalSSRC(vie_channel_, send_ssrc) == -1) { 3120 LOG_RTCERR2(GetLocalSSRC, vie_channel_, send_ssrc); 3121 return false; 3122 } 3123 if (rtp->SetLocalSSRC(channel_id, send_ssrc) == -1) { 3124 LOG_RTCERR2(SetLocalSSRC, channel_id, send_ssrc); 3125 return false; 3126 } 3127 } // Else this is the the default channel and we don't change the SSRC. 3128 3129 // Disable color enhancement since it is a bit too aggressive. 3130 if (engine()->vie()->image()->EnableColorEnhancement(channel_id, 3131 false) != 0) { 3132 LOG_RTCERR1(EnableColorEnhancement, channel_id); 3133 return false; 3134 } 3135 3136 if (!SetReceiveCodecs(channel_info.get())) { 3137 return false; 3138 } 3139 3140 int buffer_latency = 3141 options_.buffered_mode_latency.GetWithDefaultIfUnset( 3142 cricket::kBufferedModeDisabled); 3143 if (buffer_latency != cricket::kBufferedModeDisabled) { 3144 if (engine()->vie()->rtp()->SetReceiverBufferingMode( 3145 channel_id, buffer_latency) != 0) { 3146 LOG_RTCERR2(SetReceiverBufferingMode, channel_id, buffer_latency); 3147 } 3148 } 3149 3150 if (render_started_) { 3151 if (engine_->vie()->render()->StartRender(channel_id) != 0) { 3152 LOG_RTCERR1(StartRender, channel_id); 3153 return false; 3154 } 3155 } 3156 3157 // Register decoder observer for incoming framerate and bitrate. 3158 if (engine()->vie()->codec()->RegisterDecoderObserver( 3159 channel_id, *channel_info->decoder_observer()) != 0) { 3160 LOG_RTCERR1(RegisterDecoderObserver, channel_info->decoder_observer()); 3161 return false; 3162 } 3163 3164 recv_channels_[remote_ssrc_key] = channel_info.release(); 3165 return true; 3166 } 3167 3168 bool WebRtcVideoMediaChannel::ConfigureSending(int channel_id, 3169 uint32 local_ssrc_key) { 3170 // The ssrc key can be zero or correspond to an SSRC. 3171 // Make sure the default channel isn't configured more than once. 3172 if (local_ssrc_key == 0 && send_channels_.find(0) != send_channels_.end()) { 3173 return false; 3174 } 3175 // Make sure that the SSRC is not already in use. 3176 uint32 dummy_key; 3177 if (GetSendChannelKey(local_ssrc_key, &dummy_key)) { 3178 return false; 3179 } 3180 int vie_capture = 0; 3181 webrtc::ViEExternalCapture* external_capture = NULL; 3182 // Register external capture. 3183 if (engine()->vie()->capture()->AllocateExternalCaptureDevice( 3184 vie_capture, external_capture) != 0) { 3185 LOG_RTCERR0(AllocateExternalCaptureDevice); 3186 return false; 3187 } 3188 3189 // Connect external capture. 3190 if (engine()->vie()->capture()->ConnectCaptureDevice( 3191 vie_capture, channel_id) != 0) { 3192 LOG_RTCERR2(ConnectCaptureDevice, vie_capture, channel_id); 3193 return false; 3194 } 3195 talk_base::scoped_ptr<WebRtcVideoChannelSendInfo> send_channel( 3196 new WebRtcVideoChannelSendInfo(channel_id, vie_capture, 3197 external_capture, 3198 engine()->cpu_monitor())); 3199 if (engine()->vie()->base()->RegisterCpuOveruseObserver( 3200 channel_id, send_channel->overuse_observer())) { 3201 LOG_RTCERR1(RegisterCpuOveruseObserver, channel_id); 3202 return false; 3203 } 3204 send_channel->ApplyCpuOptions(options_); 3205 send_channel->SignalCpuAdaptationUnable.connect(this, 3206 &WebRtcVideoMediaChannel::OnCpuAdaptationUnable); 3207 3208 if (options_.cpu_overuse_detection.GetWithDefaultIfUnset(false)) { 3209 send_channel->SetCpuOveruseDetection(true); 3210 } 3211 3212 // Register encoder observer for outgoing framerate and bitrate. 3213 if (engine()->vie()->codec()->RegisterEncoderObserver( 3214 channel_id, *send_channel->encoder_observer()) != 0) { 3215 LOG_RTCERR1(RegisterEncoderObserver, send_channel->encoder_observer()); 3216 return false; 3217 } 3218 3219 if (!SetHeaderExtension(&webrtc::ViERTP_RTCP::SetSendTimestampOffsetStatus, 3220 channel_id, send_extensions_, kRtpTimestampOffsetHeaderExtension)) { 3221 return false; 3222 } 3223 3224 if (!SetHeaderExtension(&webrtc::ViERTP_RTCP::SetSendAbsoluteSendTimeStatus, 3225 channel_id, send_extensions_, kRtpAbsoluteSendTimeHeaderExtension)) { 3226 return false; 3227 } 3228 3229 if (options_.video_leaky_bucket.GetWithDefaultIfUnset(false)) { 3230 if (engine()->vie()->rtp()->SetTransmissionSmoothingStatus(channel_id, 3231 true) != 0) { 3232 LOG_RTCERR2(SetTransmissionSmoothingStatus, channel_id, true); 3233 return false; 3234 } 3235 } 3236 3237 int buffer_latency = 3238 options_.buffered_mode_latency.GetWithDefaultIfUnset( 3239 cricket::kBufferedModeDisabled); 3240 if (buffer_latency != cricket::kBufferedModeDisabled) { 3241 if (engine()->vie()->rtp()->SetSenderBufferingMode( 3242 channel_id, buffer_latency) != 0) { 3243 LOG_RTCERR2(SetSenderBufferingMode, channel_id, buffer_latency); 3244 } 3245 } 3246 // The remb status direction correspond to the RTP stream (and not the RTCP 3247 // stream). I.e. if send remb is enabled it means it is receiving remote 3248 // rembs and should use them to estimate bandwidth. Receive remb mean that 3249 // remb packets will be generated and that the channel should be included in 3250 // it. If remb is enabled all channels are allowed to contribute to the remb 3251 // but only receive channels will ever end up actually contributing. This 3252 // keeps the logic simple. 3253 if (engine_->vie()->rtp()->SetRembStatus(channel_id, 3254 remb_enabled_, 3255 remb_enabled_) != 0) { 3256 LOG_RTCERR3(SetRembStatus, channel_id, remb_enabled_, remb_enabled_); 3257 return false; 3258 } 3259 if (!SetNackFec(channel_id, send_red_type_, send_fec_type_, nack_enabled_)) { 3260 // Logged in SetNackFec. Don't spam the logs. 3261 return false; 3262 } 3263 3264 send_channels_[local_ssrc_key] = send_channel.release(); 3265 3266 return true; 3267 } 3268 3269 bool WebRtcVideoMediaChannel::SetNackFec(int channel_id, 3270 int red_payload_type, 3271 int fec_payload_type, 3272 bool nack_enabled) { 3273 bool enable = (red_payload_type != -1 && fec_payload_type != -1 && 3274 !InConferenceMode()); 3275 if (enable) { 3276 if (engine_->vie()->rtp()->SetHybridNACKFECStatus( 3277 channel_id, nack_enabled, red_payload_type, fec_payload_type) != 0) { 3278 LOG_RTCERR4(SetHybridNACKFECStatus, 3279 channel_id, nack_enabled, red_payload_type, fec_payload_type); 3280 return false; 3281 } 3282 LOG(LS_INFO) << "Hybrid NACK/FEC enabled for channel " << channel_id; 3283 } else { 3284 if (engine_->vie()->rtp()->SetNACKStatus(channel_id, nack_enabled) != 0) { 3285 LOG_RTCERR1(SetNACKStatus, channel_id); 3286 return false; 3287 } 3288 std::string enabled = nack_enabled ? "enabled" : "disabled"; 3289 LOG(LS_INFO) << "NACK " << enabled << " for channel " << channel_id; 3290 } 3291 return true; 3292 } 3293 3294 bool WebRtcVideoMediaChannel::SetSendCodec(const webrtc::VideoCodec& codec, 3295 int min_bitrate, 3296 int start_bitrate, 3297 int max_bitrate) { 3298 bool ret_val = true; 3299 for (SendChannelMap::iterator iter = send_channels_.begin(); 3300 iter != send_channels_.end(); ++iter) { 3301 WebRtcVideoChannelSendInfo* send_channel = iter->second; 3302 ret_val = SetSendCodec(send_channel, codec, min_bitrate, start_bitrate, 3303 max_bitrate) && ret_val; 3304 } 3305 if (ret_val) { 3306 // All SetSendCodec calls were successful. Update the global state 3307 // accordingly. 3308 send_codec_.reset(new webrtc::VideoCodec(codec)); 3309 send_min_bitrate_ = min_bitrate; 3310 send_start_bitrate_ = start_bitrate; 3311 send_max_bitrate_ = max_bitrate; 3312 } else { 3313 // At least one SetSendCodec call failed, rollback. 3314 for (SendChannelMap::iterator iter = send_channels_.begin(); 3315 iter != send_channels_.end(); ++iter) { 3316 WebRtcVideoChannelSendInfo* send_channel = iter->second; 3317 if (send_codec_) { 3318 SetSendCodec(send_channel, *send_codec_.get(), send_min_bitrate_, 3319 send_start_bitrate_, send_max_bitrate_); 3320 } 3321 } 3322 } 3323 return ret_val; 3324 } 3325 3326 bool WebRtcVideoMediaChannel::SetSendCodec( 3327 WebRtcVideoChannelSendInfo* send_channel, 3328 const webrtc::VideoCodec& codec, 3329 int min_bitrate, 3330 int start_bitrate, 3331 int max_bitrate) { 3332 if (!send_channel) { 3333 return false; 3334 } 3335 const int channel_id = send_channel->channel_id(); 3336 // Make a copy of the codec 3337 webrtc::VideoCodec target_codec = codec; 3338 target_codec.startBitrate = start_bitrate; 3339 target_codec.minBitrate = min_bitrate; 3340 target_codec.maxBitrate = max_bitrate; 3341 3342 // Set the default number of temporal layers for VP8. 3343 if (webrtc::kVideoCodecVP8 == codec.codecType) { 3344 target_codec.codecSpecific.VP8.numberOfTemporalLayers = 3345 kDefaultNumberOfTemporalLayers; 3346 3347 // Turn off the VP8 error resilience 3348 target_codec.codecSpecific.VP8.resilience = webrtc::kResilienceOff; 3349 3350 bool enable_denoising = 3351 options_.video_noise_reduction.GetWithDefaultIfUnset(false); 3352 target_codec.codecSpecific.VP8.denoisingOn = enable_denoising; 3353 } 3354 3355 // Register external encoder if codec type is supported by encoder factory. 3356 if (engine()->IsExternalEncoderCodecType(codec.codecType) && 3357 !send_channel->IsEncoderRegistered(target_codec.plType)) { 3358 webrtc::VideoEncoder* encoder = 3359 engine()->CreateExternalEncoder(codec.codecType); 3360 if (encoder) { 3361 if (engine()->vie()->ext_codec()->RegisterExternalSendCodec( 3362 channel_id, target_codec.plType, encoder, false) == 0) { 3363 send_channel->RegisterEncoder(target_codec.plType, encoder); 3364 } else { 3365 LOG_RTCERR2(RegisterExternalSendCodec, channel_id, target_codec.plName); 3366 engine()->DestroyExternalEncoder(encoder); 3367 } 3368 } 3369 } 3370 3371 // Resolution and framerate may vary for different send channels. 3372 const VideoFormat& video_format = send_channel->video_format(); 3373 UpdateVideoCodec(video_format, &target_codec); 3374 3375 if (target_codec.width == 0 && target_codec.height == 0) { 3376 const uint32 ssrc = send_channel->stream_params()->first_ssrc(); 3377 LOG(LS_INFO) << "0x0 resolution selected. Captured frames will be dropped " 3378 << "for ssrc: " << ssrc << "."; 3379 } else { 3380 MaybeChangeStartBitrate(channel_id, &target_codec); 3381 if (0 != engine()->vie()->codec()->SetSendCodec(channel_id, target_codec)) { 3382 LOG_RTCERR2(SetSendCodec, channel_id, target_codec.plName); 3383 return false; 3384 } 3385 3386 // NOTE: SetRtxSendPayloadType must be called after all simulcast SSRCs 3387 // are configured. Otherwise ssrc's configured after this point will use 3388 // the primary PT for RTX. 3389 if (send_rtx_type_ != -1 && 3390 engine()->vie()->rtp()->SetRtxSendPayloadType(channel_id, 3391 send_rtx_type_) != 0) { 3392 LOG_RTCERR2(SetRtxSendPayloadType, channel_id, send_rtx_type_); 3393 return false; 3394 } 3395 } 3396 send_channel->set_interval( 3397 cricket::VideoFormat::FpsToInterval(target_codec.maxFramerate)); 3398 return true; 3399 } 3400 3401 3402 static std::string ToString(webrtc::VideoCodecComplexity complexity) { 3403 switch (complexity) { 3404 case webrtc::kComplexityNormal: 3405 return "normal"; 3406 case webrtc::kComplexityHigh: 3407 return "high"; 3408 case webrtc::kComplexityHigher: 3409 return "higher"; 3410 case webrtc::kComplexityMax: 3411 return "max"; 3412 default: 3413 return "unknown"; 3414 } 3415 } 3416 3417 static std::string ToString(webrtc::VP8ResilienceMode resilience) { 3418 switch (resilience) { 3419 case webrtc::kResilienceOff: 3420 return "off"; 3421 case webrtc::kResilientStream: 3422 return "stream"; 3423 case webrtc::kResilientFrames: 3424 return "frames"; 3425 default: 3426 return "unknown"; 3427 } 3428 } 3429 3430 void WebRtcVideoMediaChannel::LogSendCodecChange(const std::string& reason) { 3431 webrtc::VideoCodec vie_codec; 3432 if (engine()->vie()->codec()->GetSendCodec(vie_channel_, vie_codec) != 0) { 3433 LOG_RTCERR1(GetSendCodec, vie_channel_); 3434 return; 3435 } 3436 3437 LOG(LS_INFO) << reason << " : selected video codec " 3438 << vie_codec.plName << "/" 3439 << vie_codec.width << "x" << vie_codec.height << "x" 3440 << static_cast<int>(vie_codec.maxFramerate) << "fps" 3441 << "@" << vie_codec.maxBitrate << "kbps" 3442 << " (min=" << vie_codec.minBitrate << "kbps," 3443 << " start=" << vie_codec.startBitrate << "kbps)"; 3444 LOG(LS_INFO) << "Video max quantization: " << vie_codec.qpMax; 3445 if (webrtc::kVideoCodecVP8 == vie_codec.codecType) { 3446 LOG(LS_INFO) << "VP8 number of temporal layers: " 3447 << static_cast<int>( 3448 vie_codec.codecSpecific.VP8.numberOfTemporalLayers); 3449 LOG(LS_INFO) << "VP8 options : " 3450 << "picture loss indication = " 3451 << vie_codec.codecSpecific.VP8.pictureLossIndicationOn 3452 << ", feedback mode = " 3453 << vie_codec.codecSpecific.VP8.feedbackModeOn 3454 << ", complexity = " 3455 << ToString(vie_codec.codecSpecific.VP8.complexity) 3456 << ", resilience = " 3457 << ToString(vie_codec.codecSpecific.VP8.resilience) 3458 << ", denoising = " 3459 << vie_codec.codecSpecific.VP8.denoisingOn 3460 << ", error concealment = " 3461 << vie_codec.codecSpecific.VP8.errorConcealmentOn 3462 << ", automatic resize = " 3463 << vie_codec.codecSpecific.VP8.automaticResizeOn 3464 << ", frame dropping = " 3465 << vie_codec.codecSpecific.VP8.frameDroppingOn 3466 << ", key frame interval = " 3467 << vie_codec.codecSpecific.VP8.keyFrameInterval; 3468 } 3469 3470 if (send_rtx_type_ != -1) { 3471 LOG(LS_INFO) << "RTX payload type: " << send_rtx_type_; 3472 } 3473 } 3474 3475 bool WebRtcVideoMediaChannel::SetReceiveCodecs( 3476 WebRtcVideoChannelRecvInfo* info) { 3477 int red_type = -1; 3478 int fec_type = -1; 3479 int channel_id = info->channel_id(); 3480 for (std::vector<webrtc::VideoCodec>::iterator it = receive_codecs_.begin(); 3481 it != receive_codecs_.end(); ++it) { 3482 if (it->codecType == webrtc::kVideoCodecRED) { 3483 red_type = it->plType; 3484 } else if (it->codecType == webrtc::kVideoCodecULPFEC) { 3485 fec_type = it->plType; 3486 } 3487 if (engine()->vie()->codec()->SetReceiveCodec(channel_id, *it) != 0) { 3488 LOG_RTCERR2(SetReceiveCodec, channel_id, it->plName); 3489 return false; 3490 } 3491 if (!info->IsDecoderRegistered(it->plType) && 3492 it->codecType != webrtc::kVideoCodecRED && 3493 it->codecType != webrtc::kVideoCodecULPFEC) { 3494 webrtc::VideoDecoder* decoder = 3495 engine()->CreateExternalDecoder(it->codecType); 3496 if (decoder) { 3497 if (engine()->vie()->ext_codec()->RegisterExternalReceiveCodec( 3498 channel_id, it->plType, decoder) == 0) { 3499 info->RegisterDecoder(it->plType, decoder); 3500 } else { 3501 LOG_RTCERR2(RegisterExternalReceiveCodec, channel_id, it->plName); 3502 engine()->DestroyExternalDecoder(decoder); 3503 } 3504 } 3505 } 3506 } 3507 3508 // Start receiving packets if at least one receive codec has been set. 3509 if (!receive_codecs_.empty()) { 3510 if (engine()->vie()->base()->StartReceive(channel_id) != 0) { 3511 LOG_RTCERR1(StartReceive, channel_id); 3512 return false; 3513 } 3514 } 3515 return true; 3516 } 3517 3518 int WebRtcVideoMediaChannel::GetRecvChannelNum(uint32 ssrc) { 3519 if (ssrc == first_receive_ssrc_) { 3520 return vie_channel_; 3521 } 3522 RecvChannelMap::iterator it = recv_channels_.find(ssrc); 3523 return (it != recv_channels_.end()) ? it->second->channel_id() : -1; 3524 } 3525 3526 // If the new frame size is different from the send codec size we set on vie, 3527 // we need to reset the send codec on vie. 3528 // The new send codec size should not exceed send_codec_ which is controlled 3529 // only by the 'jec' logic. 3530 bool WebRtcVideoMediaChannel::MaybeResetVieSendCodec( 3531 WebRtcVideoChannelSendInfo* send_channel, 3532 int new_width, 3533 int new_height, 3534 bool is_screencast, 3535 bool* reset) { 3536 if (reset) { 3537 *reset = false; 3538 } 3539 ASSERT(send_codec_.get() != NULL); 3540 3541 webrtc::VideoCodec target_codec = *send_codec_.get(); 3542 const VideoFormat& video_format = send_channel->video_format(); 3543 UpdateVideoCodec(video_format, &target_codec); 3544 3545 // Vie send codec size should not exceed target_codec. 3546 int target_width = new_width; 3547 int target_height = new_height; 3548 if (!is_screencast && 3549 (new_width > target_codec.width || new_height > target_codec.height)) { 3550 target_width = target_codec.width; 3551 target_height = target_codec.height; 3552 } 3553 3554 // Get current vie codec. 3555 webrtc::VideoCodec vie_codec; 3556 const int channel_id = send_channel->channel_id(); 3557 if (engine()->vie()->codec()->GetSendCodec(channel_id, vie_codec) != 0) { 3558 LOG_RTCERR1(GetSendCodec, channel_id); 3559 return false; 3560 } 3561 const int cur_width = vie_codec.width; 3562 const int cur_height = vie_codec.height; 3563 3564 // Only reset send codec when there is a size change. Additionally, 3565 // automatic resize needs to be turned off when screencasting and on when 3566 // not screencasting. 3567 // Don't allow automatic resizing for screencasting. 3568 bool automatic_resize = !is_screencast; 3569 // Turn off VP8 frame dropping when screensharing as the current model does 3570 // not work well at low fps. 3571 bool vp8_frame_dropping = !is_screencast; 3572 // Disable denoising for screencasting. 3573 bool enable_denoising = 3574 options_.video_noise_reduction.GetWithDefaultIfUnset(false); 3575 bool denoising = !is_screencast && enable_denoising; 3576 bool reset_send_codec = 3577 target_width != cur_width || target_height != cur_height || 3578 automatic_resize != vie_codec.codecSpecific.VP8.automaticResizeOn || 3579 denoising != vie_codec.codecSpecific.VP8.denoisingOn || 3580 vp8_frame_dropping != vie_codec.codecSpecific.VP8.frameDroppingOn; 3581 3582 if (reset_send_codec) { 3583 // Set the new codec on vie. 3584 vie_codec.width = target_width; 3585 vie_codec.height = target_height; 3586 vie_codec.maxFramerate = target_codec.maxFramerate; 3587 vie_codec.startBitrate = target_codec.startBitrate; 3588 vie_codec.codecSpecific.VP8.automaticResizeOn = automatic_resize; 3589 vie_codec.codecSpecific.VP8.denoisingOn = denoising; 3590 vie_codec.codecSpecific.VP8.frameDroppingOn = vp8_frame_dropping; 3591 // TODO(mflodman): Remove 'is_screencast' check when screen cast settings 3592 // are treated correctly in WebRTC. 3593 if (!is_screencast) 3594 MaybeChangeStartBitrate(channel_id, &vie_codec); 3595 3596 if (engine()->vie()->codec()->SetSendCodec(channel_id, vie_codec) != 0) { 3597 LOG_RTCERR1(SetSendCodec, channel_id); 3598 return false; 3599 } 3600 if (reset) { 3601 *reset = true; 3602 } 3603 LogSendCodecChange("Capture size changed"); 3604 } 3605 3606 return true; 3607 } 3608 3609 void WebRtcVideoMediaChannel::MaybeChangeStartBitrate( 3610 int channel_id, webrtc::VideoCodec* video_codec) { 3611 if (video_codec->startBitrate < video_codec->minBitrate) { 3612 video_codec->startBitrate = video_codec->minBitrate; 3613 } else if (video_codec->startBitrate > video_codec->maxBitrate) { 3614 video_codec->startBitrate = video_codec->maxBitrate; 3615 } 3616 3617 // Use a previous target bitrate, if there is one. 3618 unsigned int current_target_bitrate = 0; 3619 if (engine()->vie()->codec()->GetCodecTargetBitrate( 3620 channel_id, ¤t_target_bitrate) == 0) { 3621 // Convert to kbps. 3622 current_target_bitrate /= 1000; 3623 if (current_target_bitrate > video_codec->maxBitrate) { 3624 current_target_bitrate = video_codec->maxBitrate; 3625 } 3626 if (current_target_bitrate > video_codec->startBitrate) { 3627 video_codec->startBitrate = current_target_bitrate; 3628 } 3629 } 3630 } 3631 3632 void WebRtcVideoMediaChannel::OnMessage(talk_base::Message* msg) { 3633 FlushBlackFrameData* black_frame_data = 3634 static_cast<FlushBlackFrameData*>(msg->pdata); 3635 FlushBlackFrame(black_frame_data->ssrc, black_frame_data->timestamp); 3636 delete black_frame_data; 3637 } 3638 3639 int WebRtcVideoMediaChannel::SendPacket(int channel, const void* data, 3640 int len) { 3641 talk_base::Buffer packet(data, len, kMaxRtpPacketLen); 3642 return MediaChannel::SendPacket(&packet) ? len : -1; 3643 } 3644 3645 int WebRtcVideoMediaChannel::SendRTCPPacket(int channel, 3646 const void* data, 3647 int len) { 3648 talk_base::Buffer packet(data, len, kMaxRtpPacketLen); 3649 return MediaChannel::SendRtcp(&packet) ? len : -1; 3650 } 3651 3652 void WebRtcVideoMediaChannel::QueueBlackFrame(uint32 ssrc, int64 timestamp, 3653 int framerate) { 3654 if (timestamp) { 3655 FlushBlackFrameData* black_frame_data = new FlushBlackFrameData( 3656 ssrc, 3657 timestamp); 3658 const int delay_ms = static_cast<int>( 3659 2 * cricket::VideoFormat::FpsToInterval(framerate) * 3660 talk_base::kNumMillisecsPerSec / talk_base::kNumNanosecsPerSec); 3661 worker_thread()->PostDelayed(delay_ms, this, 0, black_frame_data); 3662 } 3663 } 3664 3665 void WebRtcVideoMediaChannel::FlushBlackFrame(uint32 ssrc, int64 timestamp) { 3666 WebRtcVideoChannelSendInfo* send_channel = GetSendChannel(ssrc); 3667 if (!send_channel) { 3668 return; 3669 } 3670 talk_base::scoped_ptr<const VideoFrame> black_frame_ptr; 3671 3672 const WebRtcLocalStreamInfo* channel_stream_info = 3673 send_channel->local_stream_info(); 3674 int64 last_frame_time_stamp = channel_stream_info->time_stamp(); 3675 if (last_frame_time_stamp == timestamp) { 3676 size_t last_frame_width = 0; 3677 size_t last_frame_height = 0; 3678 int64 last_frame_elapsed_time = 0; 3679 channel_stream_info->GetLastFrameInfo(&last_frame_width, &last_frame_height, 3680 &last_frame_elapsed_time); 3681 if (!last_frame_width || !last_frame_height) { 3682 return; 3683 } 3684 WebRtcVideoFrame black_frame; 3685 // Black frame is not screencast. 3686 const bool screencasting = false; 3687 const int64 timestamp_delta = send_channel->interval(); 3688 if (!black_frame.InitToBlack(send_codec_->width, send_codec_->height, 1, 1, 3689 last_frame_elapsed_time + timestamp_delta, 3690 last_frame_time_stamp + timestamp_delta) || 3691 !SendFrame(send_channel, &black_frame, screencasting)) { 3692 LOG(LS_ERROR) << "Failed to send black frame."; 3693 } 3694 } 3695 } 3696 3697 void WebRtcVideoMediaChannel::OnCpuAdaptationUnable() { 3698 // ssrc is hardcoded to 0. This message is based on a system wide issue, 3699 // so finding which ssrc caused it doesn't matter. 3700 SignalMediaError(0, VideoMediaChannel::ERROR_REC_CPU_MAX_CANT_DOWNGRADE); 3701 } 3702 3703 void WebRtcVideoMediaChannel::SetNetworkTransmissionState( 3704 bool is_transmitting) { 3705 LOG(LS_INFO) << "SetNetworkTransmissionState: " << is_transmitting; 3706 for (SendChannelMap::iterator iter = send_channels_.begin(); 3707 iter != send_channels_.end(); ++iter) { 3708 WebRtcVideoChannelSendInfo* send_channel = iter->second; 3709 int channel_id = send_channel->channel_id(); 3710 engine_->vie()->network()->SetNetworkTransmissionState(channel_id, 3711 is_transmitting); 3712 } 3713 } 3714 3715 bool WebRtcVideoMediaChannel::SetHeaderExtension(ExtensionSetterFunction setter, 3716 int channel_id, const RtpHeaderExtension* extension) { 3717 bool enable = false; 3718 int id = 0; 3719 if (extension) { 3720 enable = true; 3721 id = extension->id; 3722 } 3723 if ((engine_->vie()->rtp()->*setter)(channel_id, enable, id) != 0) { 3724 LOG_RTCERR4(*setter, extension->uri, channel_id, enable, id); 3725 return false; 3726 } 3727 return true; 3728 } 3729 3730 bool WebRtcVideoMediaChannel::SetHeaderExtension(ExtensionSetterFunction setter, 3731 int channel_id, const std::vector<RtpHeaderExtension>& extensions, 3732 const char header_extension_uri[]) { 3733 const RtpHeaderExtension* extension = FindHeaderExtension(extensions, 3734 header_extension_uri); 3735 return SetHeaderExtension(setter, channel_id, extension); 3736 } 3737 3738 bool WebRtcVideoMediaChannel::SetLocalRtxSsrc(int channel_id, 3739 const StreamParams& send_params, 3740 uint32 primary_ssrc, 3741 int stream_idx) { 3742 uint32 rtx_ssrc = 0; 3743 bool has_rtx = send_params.GetFidSsrc(primary_ssrc, &rtx_ssrc); 3744 if (has_rtx && engine()->vie()->rtp()->SetLocalSSRC( 3745 channel_id, rtx_ssrc, webrtc::kViEStreamTypeRtx, stream_idx) != 0) { 3746 LOG_RTCERR4(SetLocalSSRC, channel_id, rtx_ssrc, 3747 webrtc::kViEStreamTypeRtx, stream_idx); 3748 return false; 3749 } 3750 return true; 3751 } 3752 3753 void WebRtcVideoMediaChannel::MaybeConnectCapturer(VideoCapturer* capturer) { 3754 if (capturer != NULL && GetSendChannelNum(capturer) == 1) { 3755 capturer->SignalVideoFrame.connect(this, 3756 &WebRtcVideoMediaChannel::SendFrame); 3757 } 3758 } 3759 3760 void WebRtcVideoMediaChannel::MaybeDisconnectCapturer(VideoCapturer* capturer) { 3761 if (capturer != NULL && GetSendChannelNum(capturer) == 1) { 3762 capturer->SignalVideoFrame.disconnect(this); 3763 } 3764 } 3765 3766 } // namespace cricket 3767 3768 #endif // HAVE_WEBRTC_VIDEO 3769