1 /* 2 * libjingle 3 * Copyright 2004 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #ifndef TALK_MEDIA_BASE_MEDIACHANNEL_H_ 29 #define TALK_MEDIA_BASE_MEDIACHANNEL_H_ 30 31 #include <string> 32 #include <vector> 33 34 #include "talk/base/basictypes.h" 35 #include "talk/base/buffer.h" 36 #include "talk/base/logging.h" 37 #include "talk/base/sigslot.h" 38 #include "talk/base/socket.h" 39 #include "talk/base/window.h" 40 #include "talk/media/base/codec.h" 41 #include "talk/media/base/constants.h" 42 #include "talk/media/base/streamparams.h" 43 // TODO(juberti): re-evaluate this include 44 #include "talk/session/media/audiomonitor.h" 45 46 namespace talk_base { 47 class Buffer; 48 class RateLimiter; 49 class Timing; 50 } 51 52 namespace cricket { 53 54 class AudioRenderer; 55 struct RtpHeader; 56 class ScreencastId; 57 struct VideoFormat; 58 class VideoCapturer; 59 class VideoRenderer; 60 61 const int kMinRtpHeaderExtensionId = 1; 62 const int kMaxRtpHeaderExtensionId = 255; 63 const int kScreencastDefaultFps = 5; 64 65 // Used in AudioOptions and VideoOptions to signify "unset" values. 66 template <class T> 67 class Settable { 68 public: 69 Settable() : set_(false), val_() {} 70 explicit Settable(T val) : set_(true), val_(val) {} 71 72 bool IsSet() const { 73 return set_; 74 } 75 76 bool Get(T* out) const { 77 *out = val_; 78 return set_; 79 } 80 81 T GetWithDefaultIfUnset(const T& default_value) const { 82 return set_ ? val_ : default_value; 83 } 84 85 virtual void Set(T val) { 86 set_ = true; 87 val_ = val; 88 } 89 90 void Clear() { 91 Set(T()); 92 set_ = false; 93 } 94 95 void SetFrom(const Settable<T>& o) { 96 // Set this value based on the value of o, iff o is set. If this value is 97 // set and o is unset, the current value will be unchanged. 98 T val; 99 if (o.Get(&val)) { 100 Set(val); 101 } 102 } 103 104 std::string ToString() const { 105 return set_ ? talk_base::ToString(val_) : ""; 106 } 107 108 bool operator==(const Settable<T>& o) const { 109 // Equal if both are unset with any value or both set with the same value. 110 return (set_ == o.set_) && (!set_ || (val_ == o.val_)); 111 } 112 113 bool operator!=(const Settable<T>& o) const { 114 return !operator==(o); 115 } 116 117 protected: 118 void InitializeValue(const T &val) { 119 val_ = val; 120 } 121 122 private: 123 bool set_; 124 T val_; 125 }; 126 127 class SettablePercent : public Settable<float> { 128 public: 129 virtual void Set(float val) { 130 if (val < 0) { 131 val = 0; 132 } 133 if (val > 1.0) { 134 val = 1.0; 135 } 136 Settable<float>::Set(val); 137 } 138 }; 139 140 template <class T> 141 static std::string ToStringIfSet(const char* key, const Settable<T>& val) { 142 std::string str; 143 if (val.IsSet()) { 144 str = key; 145 str += ": "; 146 str += val.ToString(); 147 str += ", "; 148 } 149 return str; 150 } 151 152 // Options that can be applied to a VoiceMediaChannel or a VoiceMediaEngine. 153 // Used to be flags, but that makes it hard to selectively apply options. 154 // We are moving all of the setting of options to structs like this, 155 // but some things currently still use flags. 156 struct AudioOptions { 157 void SetAll(const AudioOptions& change) { 158 echo_cancellation.SetFrom(change.echo_cancellation); 159 auto_gain_control.SetFrom(change.auto_gain_control); 160 noise_suppression.SetFrom(change.noise_suppression); 161 highpass_filter.SetFrom(change.highpass_filter); 162 stereo_swapping.SetFrom(change.stereo_swapping); 163 typing_detection.SetFrom(change.typing_detection); 164 conference_mode.SetFrom(change.conference_mode); 165 adjust_agc_delta.SetFrom(change.adjust_agc_delta); 166 experimental_agc.SetFrom(change.experimental_agc); 167 experimental_aec.SetFrom(change.experimental_aec); 168 aec_dump.SetFrom(change.aec_dump); 169 } 170 171 bool operator==(const AudioOptions& o) const { 172 return echo_cancellation == o.echo_cancellation && 173 auto_gain_control == o.auto_gain_control && 174 noise_suppression == o.noise_suppression && 175 highpass_filter == o.highpass_filter && 176 stereo_swapping == o.stereo_swapping && 177 typing_detection == o.typing_detection && 178 conference_mode == o.conference_mode && 179 experimental_agc == o.experimental_agc && 180 experimental_aec == o.experimental_aec && 181 adjust_agc_delta == o.adjust_agc_delta && 182 aec_dump == o.aec_dump; 183 } 184 185 std::string ToString() const { 186 std::ostringstream ost; 187 ost << "AudioOptions {"; 188 ost << ToStringIfSet("aec", echo_cancellation); 189 ost << ToStringIfSet("agc", auto_gain_control); 190 ost << ToStringIfSet("ns", noise_suppression); 191 ost << ToStringIfSet("hf", highpass_filter); 192 ost << ToStringIfSet("swap", stereo_swapping); 193 ost << ToStringIfSet("typing", typing_detection); 194 ost << ToStringIfSet("conference", conference_mode); 195 ost << ToStringIfSet("agc_delta", adjust_agc_delta); 196 ost << ToStringIfSet("experimental_agc", experimental_agc); 197 ost << ToStringIfSet("experimental_aec", experimental_aec); 198 ost << ToStringIfSet("aec_dump", aec_dump); 199 ost << "}"; 200 return ost.str(); 201 } 202 203 // Audio processing that attempts to filter away the output signal from 204 // later inbound pickup. 205 Settable<bool> echo_cancellation; 206 // Audio processing to adjust the sensitivity of the local mic dynamically. 207 Settable<bool> auto_gain_control; 208 // Audio processing to filter out background noise. 209 Settable<bool> noise_suppression; 210 // Audio processing to remove background noise of lower frequencies. 211 Settable<bool> highpass_filter; 212 // Audio processing to swap the left and right channels. 213 Settable<bool> stereo_swapping; 214 // Audio processing to detect typing. 215 Settable<bool> typing_detection; 216 Settable<bool> conference_mode; 217 Settable<int> adjust_agc_delta; 218 Settable<bool> experimental_agc; 219 Settable<bool> experimental_aec; 220 Settable<bool> aec_dump; 221 }; 222 223 // Options that can be applied to a VideoMediaChannel or a VideoMediaEngine. 224 // Used to be flags, but that makes it hard to selectively apply options. 225 // We are moving all of the setting of options to structs like this, 226 // but some things currently still use flags. 227 struct VideoOptions { 228 VideoOptions() { 229 process_adaptation_threshhold.Set(kProcessCpuThreshold); 230 system_low_adaptation_threshhold.Set(kLowSystemCpuThreshold); 231 system_high_adaptation_threshhold.Set(kHighSystemCpuThreshold); 232 } 233 234 void SetAll(const VideoOptions& change) { 235 adapt_input_to_encoder.SetFrom(change.adapt_input_to_encoder); 236 adapt_input_to_cpu_usage.SetFrom(change.adapt_input_to_cpu_usage); 237 adapt_cpu_with_smoothing.SetFrom(change.adapt_cpu_with_smoothing); 238 adapt_view_switch.SetFrom(change.adapt_view_switch); 239 video_noise_reduction.SetFrom(change.video_noise_reduction); 240 video_three_layers.SetFrom(change.video_three_layers); 241 video_enable_camera_list.SetFrom(change.video_enable_camera_list); 242 video_one_layer_screencast.SetFrom(change.video_one_layer_screencast); 243 video_high_bitrate.SetFrom(change.video_high_bitrate); 244 video_watermark.SetFrom(change.video_watermark); 245 video_temporal_layer_screencast.SetFrom( 246 change.video_temporal_layer_screencast); 247 video_leaky_bucket.SetFrom(change.video_leaky_bucket); 248 conference_mode.SetFrom(change.conference_mode); 249 process_adaptation_threshhold.SetFrom(change.process_adaptation_threshhold); 250 system_low_adaptation_threshhold.SetFrom( 251 change.system_low_adaptation_threshhold); 252 system_high_adaptation_threshhold.SetFrom( 253 change.system_high_adaptation_threshhold); 254 buffered_mode_latency.SetFrom(change.buffered_mode_latency); 255 } 256 257 bool operator==(const VideoOptions& o) const { 258 return adapt_input_to_encoder == o.adapt_input_to_encoder && 259 adapt_input_to_cpu_usage == o.adapt_input_to_cpu_usage && 260 adapt_cpu_with_smoothing == o.adapt_cpu_with_smoothing && 261 adapt_view_switch == o.adapt_view_switch && 262 video_noise_reduction == o.video_noise_reduction && 263 video_three_layers == o.video_three_layers && 264 video_enable_camera_list == o.video_enable_camera_list && 265 video_one_layer_screencast == o.video_one_layer_screencast && 266 video_high_bitrate == o.video_high_bitrate && 267 video_watermark == o.video_watermark && 268 video_temporal_layer_screencast == o.video_temporal_layer_screencast && 269 video_leaky_bucket == o.video_leaky_bucket && 270 conference_mode == o.conference_mode && 271 process_adaptation_threshhold == o.process_adaptation_threshhold && 272 system_low_adaptation_threshhold == 273 o.system_low_adaptation_threshhold && 274 system_high_adaptation_threshhold == 275 o.system_high_adaptation_threshhold && 276 buffered_mode_latency == o.buffered_mode_latency; 277 } 278 279 std::string ToString() const { 280 std::ostringstream ost; 281 ost << "VideoOptions {"; 282 ost << ToStringIfSet("encoder adaption", adapt_input_to_encoder); 283 ost << ToStringIfSet("cpu adaption", adapt_input_to_cpu_usage); 284 ost << ToStringIfSet("cpu adaptation smoothing", adapt_cpu_with_smoothing); 285 ost << ToStringIfSet("adapt view switch", adapt_view_switch); 286 ost << ToStringIfSet("noise reduction", video_noise_reduction); 287 ost << ToStringIfSet("3 layers", video_three_layers); 288 ost << ToStringIfSet("camera list", video_enable_camera_list); 289 ost << ToStringIfSet("1 layer screencast", 290 video_one_layer_screencast); 291 ost << ToStringIfSet("high bitrate", video_high_bitrate); 292 ost << ToStringIfSet("watermark", video_watermark); 293 ost << ToStringIfSet("video temporal layer screencast", 294 video_temporal_layer_screencast); 295 ost << ToStringIfSet("leaky bucket", video_leaky_bucket); 296 ost << ToStringIfSet("conference mode", conference_mode); 297 ost << ToStringIfSet("process", process_adaptation_threshhold); 298 ost << ToStringIfSet("low", system_low_adaptation_threshhold); 299 ost << ToStringIfSet("high", system_high_adaptation_threshhold); 300 ost << ToStringIfSet("buffered mode latency", buffered_mode_latency); 301 ost << "}"; 302 return ost.str(); 303 } 304 305 // Encoder adaption, which is the gd callback in LMI, and TBA in WebRTC. 306 Settable<bool> adapt_input_to_encoder; 307 // Enable CPU adaptation? 308 Settable<bool> adapt_input_to_cpu_usage; 309 // Enable CPU adaptation smoothing? 310 Settable<bool> adapt_cpu_with_smoothing; 311 // Enable Adapt View Switch? 312 Settable<bool> adapt_view_switch; 313 // Enable denoising? 314 Settable<bool> video_noise_reduction; 315 // Experimental: Enable multi layer? 316 Settable<bool> video_three_layers; 317 // Experimental: Enable camera list? 318 Settable<bool> video_enable_camera_list; 319 // Experimental: Enable one layer screencast? 320 Settable<bool> video_one_layer_screencast; 321 // Experimental: Enable WebRtc higher bitrate? 322 Settable<bool> video_high_bitrate; 323 // Experimental: Add watermark to the rendered video image. 324 Settable<bool> video_watermark; 325 // Experimental: Enable WebRTC layered screencast. 326 Settable<bool> video_temporal_layer_screencast; 327 // Enable WebRTC leaky bucket when sending media packets. 328 Settable<bool> video_leaky_bucket; 329 // Use conference mode? 330 Settable<bool> conference_mode; 331 // Threshhold for process cpu adaptation. (Process limit) 332 SettablePercent process_adaptation_threshhold; 333 // Low threshhold for cpu adaptation. (Adapt up) 334 SettablePercent system_low_adaptation_threshhold; 335 // High threshhold for cpu adaptation. (Adapt down) 336 SettablePercent system_high_adaptation_threshhold; 337 // Specify buffered mode latency in milliseconds. 338 Settable<int> buffered_mode_latency; 339 }; 340 341 // A class for playing out soundclips. 342 class SoundclipMedia { 343 public: 344 enum SoundclipFlags { 345 SF_LOOP = 1, 346 }; 347 348 virtual ~SoundclipMedia() {} 349 350 // Plays a sound out to the speakers with the given audio stream. The stream 351 // must be 16-bit little-endian 16 kHz PCM. If a stream is already playing 352 // on this SoundclipMedia, it is stopped. If clip is NULL, nothing is played. 353 // Returns whether it was successful. 354 virtual bool PlaySound(const char *clip, int len, int flags) = 0; 355 }; 356 357 struct RtpHeaderExtension { 358 RtpHeaderExtension() : id(0) {} 359 RtpHeaderExtension(const std::string& u, int i) : uri(u), id(i) {} 360 std::string uri; 361 int id; 362 // TODO(juberti): SendRecv direction; 363 364 bool operator==(const RtpHeaderExtension& ext) const { 365 // id is a reserved word in objective-c. Therefore the id attribute has to 366 // be a fully qualified name in order to compile on IOS. 367 return this->id == ext.id && 368 uri == ext.uri; 369 } 370 }; 371 372 // Returns the named header extension if found among all extensions, NULL 373 // otherwise. 374 inline const RtpHeaderExtension* FindHeaderExtension( 375 const std::vector<RtpHeaderExtension>& extensions, 376 const std::string& name) { 377 for (std::vector<RtpHeaderExtension>::const_iterator it = extensions.begin(); 378 it != extensions.end(); ++it) { 379 if (it->uri == name) 380 return &(*it); 381 } 382 return NULL; 383 } 384 385 enum MediaChannelOptions { 386 // Tune the stream for conference mode. 387 OPT_CONFERENCE = 0x0001 388 }; 389 390 enum VoiceMediaChannelOptions { 391 // Tune the audio stream for vcs with different target levels. 392 OPT_AGC_MINUS_10DB = 0x80000000 393 }; 394 395 // DTMF flags to control if a DTMF tone should be played and/or sent. 396 enum DtmfFlags { 397 DF_PLAY = 0x01, 398 DF_SEND = 0x02, 399 }; 400 401 class MediaChannel : public sigslot::has_slots<> { 402 public: 403 class NetworkInterface { 404 public: 405 enum SocketType { ST_RTP, ST_RTCP }; 406 virtual bool SendPacket(talk_base::Buffer* packet) = 0; 407 virtual bool SendRtcp(talk_base::Buffer* packet) = 0; 408 virtual int SetOption(SocketType type, talk_base::Socket::Option opt, 409 int option) = 0; 410 virtual ~NetworkInterface() {} 411 }; 412 413 MediaChannel() : network_interface_(NULL) {} 414 virtual ~MediaChannel() {} 415 416 // Sets the abstract interface class for sending RTP/RTCP data. 417 virtual void SetInterface(NetworkInterface *iface) { 418 talk_base::CritScope cs(&network_interface_crit_); 419 network_interface_ = iface; 420 } 421 422 // Called when a RTP packet is received. 423 virtual void OnPacketReceived(talk_base::Buffer* packet) = 0; 424 // Called when a RTCP packet is received. 425 virtual void OnRtcpReceived(talk_base::Buffer* packet) = 0; 426 // Called when the socket's ability to send has changed. 427 virtual void OnReadyToSend(bool ready) = 0; 428 // Creates a new outgoing media stream with SSRCs and CNAME as described 429 // by sp. 430 virtual bool AddSendStream(const StreamParams& sp) = 0; 431 // Removes an outgoing media stream. 432 // ssrc must be the first SSRC of the media stream if the stream uses 433 // multiple SSRCs. 434 virtual bool RemoveSendStream(uint32 ssrc) = 0; 435 // Creates a new incoming media stream with SSRCs and CNAME as described 436 // by sp. 437 virtual bool AddRecvStream(const StreamParams& sp) = 0; 438 // Removes an incoming media stream. 439 // ssrc must be the first SSRC of the media stream if the stream uses 440 // multiple SSRCs. 441 virtual bool RemoveRecvStream(uint32 ssrc) = 0; 442 443 // Mutes the channel. 444 virtual bool MuteStream(uint32 ssrc, bool on) = 0; 445 446 // Sets the RTP extension headers and IDs to use when sending RTP. 447 virtual bool SetRecvRtpHeaderExtensions( 448 const std::vector<RtpHeaderExtension>& extensions) = 0; 449 virtual bool SetSendRtpHeaderExtensions( 450 const std::vector<RtpHeaderExtension>& extensions) = 0; 451 // Sets the rate control to use when sending data. 452 virtual bool SetSendBandwidth(bool autobw, int bps) = 0; 453 454 // Base method to send packet using NetworkInterface. 455 bool SendPacket(talk_base::Buffer* packet) { 456 return DoSendPacket(packet, false); 457 } 458 459 bool SendRtcp(talk_base::Buffer* packet) { 460 return DoSendPacket(packet, true); 461 } 462 463 int SetOption(NetworkInterface::SocketType type, 464 talk_base::Socket::Option opt, 465 int option) { 466 talk_base::CritScope cs(&network_interface_crit_); 467 if (!network_interface_) 468 return -1; 469 470 return network_interface_->SetOption(type, opt, option); 471 } 472 473 private: 474 bool DoSendPacket(talk_base::Buffer* packet, bool rtcp) { 475 talk_base::CritScope cs(&network_interface_crit_); 476 if (!network_interface_) 477 return false; 478 479 return (!rtcp) ? network_interface_->SendPacket(packet) : 480 network_interface_->SendRtcp(packet); 481 } 482 483 // |network_interface_| can be accessed from the worker_thread and 484 // from any MediaEngine threads. This critical section is to protect accessing 485 // of network_interface_ object. 486 talk_base::CriticalSection network_interface_crit_; 487 NetworkInterface* network_interface_; 488 }; 489 490 enum SendFlags { 491 SEND_NOTHING, 492 SEND_RINGBACKTONE, 493 SEND_MICROPHONE 494 }; 495 496 struct VoiceSenderInfo { 497 VoiceSenderInfo() 498 : ssrc(0), 499 bytes_sent(0), 500 packets_sent(0), 501 packets_lost(0), 502 fraction_lost(0.0), 503 ext_seqnum(0), 504 rtt_ms(0), 505 jitter_ms(0), 506 audio_level(0), 507 aec_quality_min(0.0), 508 echo_delay_median_ms(0), 509 echo_delay_std_ms(0), 510 echo_return_loss(0), 511 echo_return_loss_enhancement(0) { 512 } 513 514 uint32 ssrc; 515 std::string codec_name; 516 int64 bytes_sent; 517 int packets_sent; 518 int packets_lost; 519 float fraction_lost; 520 int ext_seqnum; 521 int rtt_ms; 522 int jitter_ms; 523 int audio_level; 524 float aec_quality_min; 525 int echo_delay_median_ms; 526 int echo_delay_std_ms; 527 int echo_return_loss; 528 int echo_return_loss_enhancement; 529 }; 530 531 struct VoiceReceiverInfo { 532 VoiceReceiverInfo() 533 : ssrc(0), 534 bytes_rcvd(0), 535 packets_rcvd(0), 536 packets_lost(0), 537 fraction_lost(0.0), 538 ext_seqnum(0), 539 jitter_ms(0), 540 jitter_buffer_ms(0), 541 jitter_buffer_preferred_ms(0), 542 delay_estimate_ms(0), 543 audio_level(0), 544 expand_rate(0) { 545 } 546 547 uint32 ssrc; 548 int64 bytes_rcvd; 549 int packets_rcvd; 550 int packets_lost; 551 float fraction_lost; 552 int ext_seqnum; 553 int jitter_ms; 554 int jitter_buffer_ms; 555 int jitter_buffer_preferred_ms; 556 int delay_estimate_ms; 557 int audio_level; 558 // fraction of synthesized speech inserted through pre-emptive expansion 559 float expand_rate; 560 }; 561 562 struct VideoSenderInfo { 563 VideoSenderInfo() 564 : bytes_sent(0), 565 packets_sent(0), 566 packets_cached(0), 567 packets_lost(0), 568 fraction_lost(0.0), 569 firs_rcvd(0), 570 nacks_rcvd(0), 571 rtt_ms(0), 572 frame_width(0), 573 frame_height(0), 574 framerate_input(0), 575 framerate_sent(0), 576 nominal_bitrate(0), 577 preferred_bitrate(0), 578 adapt_reason(0) { 579 } 580 581 std::vector<uint32> ssrcs; 582 std::vector<SsrcGroup> ssrc_groups; 583 std::string codec_name; 584 int64 bytes_sent; 585 int packets_sent; 586 int packets_cached; 587 int packets_lost; 588 float fraction_lost; 589 int firs_rcvd; 590 int nacks_rcvd; 591 int rtt_ms; 592 int frame_width; 593 int frame_height; 594 int framerate_input; 595 int framerate_sent; 596 int nominal_bitrate; 597 int preferred_bitrate; 598 int adapt_reason; 599 }; 600 601 struct VideoReceiverInfo { 602 VideoReceiverInfo() 603 : bytes_rcvd(0), 604 packets_rcvd(0), 605 packets_lost(0), 606 packets_concealed(0), 607 fraction_lost(0.0), 608 firs_sent(0), 609 nacks_sent(0), 610 frame_width(0), 611 frame_height(0), 612 framerate_rcvd(0), 613 framerate_decoded(0), 614 framerate_output(0), 615 framerate_render_input(0), 616 framerate_render_output(0) { 617 } 618 619 std::vector<uint32> ssrcs; 620 std::vector<SsrcGroup> ssrc_groups; 621 int64 bytes_rcvd; 622 // vector<int> layer_bytes_rcvd; 623 int packets_rcvd; 624 int packets_lost; 625 int packets_concealed; 626 float fraction_lost; 627 int firs_sent; 628 int nacks_sent; 629 int frame_width; 630 int frame_height; 631 int framerate_rcvd; 632 int framerate_decoded; 633 int framerate_output; 634 // Framerate as sent to the renderer. 635 int framerate_render_input; 636 // Framerate that the renderer reports. 637 int framerate_render_output; 638 }; 639 640 struct DataSenderInfo { 641 DataSenderInfo() 642 : ssrc(0), 643 bytes_sent(0), 644 packets_sent(0) { 645 } 646 647 uint32 ssrc; 648 std::string codec_name; 649 int64 bytes_sent; 650 int packets_sent; 651 }; 652 653 struct DataReceiverInfo { 654 DataReceiverInfo() 655 : ssrc(0), 656 bytes_rcvd(0), 657 packets_rcvd(0) { 658 } 659 660 uint32 ssrc; 661 int64 bytes_rcvd; 662 int packets_rcvd; 663 }; 664 665 struct BandwidthEstimationInfo { 666 BandwidthEstimationInfo() 667 : available_send_bandwidth(0), 668 available_recv_bandwidth(0), 669 target_enc_bitrate(0), 670 actual_enc_bitrate(0), 671 retransmit_bitrate(0), 672 transmit_bitrate(0), 673 bucket_delay(0) { 674 } 675 676 int available_send_bandwidth; 677 int available_recv_bandwidth; 678 int target_enc_bitrate; 679 int actual_enc_bitrate; 680 int retransmit_bitrate; 681 int transmit_bitrate; 682 int bucket_delay; 683 }; 684 685 struct VoiceMediaInfo { 686 void Clear() { 687 senders.clear(); 688 receivers.clear(); 689 } 690 std::vector<VoiceSenderInfo> senders; 691 std::vector<VoiceReceiverInfo> receivers; 692 }; 693 694 struct VideoMediaInfo { 695 void Clear() { 696 senders.clear(); 697 receivers.clear(); 698 bw_estimations.clear(); 699 } 700 std::vector<VideoSenderInfo> senders; 701 std::vector<VideoReceiverInfo> receivers; 702 std::vector<BandwidthEstimationInfo> bw_estimations; 703 }; 704 705 struct DataMediaInfo { 706 void Clear() { 707 senders.clear(); 708 receivers.clear(); 709 } 710 std::vector<DataSenderInfo> senders; 711 std::vector<DataReceiverInfo> receivers; 712 }; 713 714 class VoiceMediaChannel : public MediaChannel { 715 public: 716 enum Error { 717 ERROR_NONE = 0, // No error. 718 ERROR_OTHER, // Other errors. 719 ERROR_REC_DEVICE_OPEN_FAILED = 100, // Could not open mic. 720 ERROR_REC_DEVICE_MUTED, // Mic was muted by OS. 721 ERROR_REC_DEVICE_SILENT, // No background noise picked up. 722 ERROR_REC_DEVICE_SATURATION, // Mic input is clipping. 723 ERROR_REC_DEVICE_REMOVED, // Mic was removed while active. 724 ERROR_REC_RUNTIME_ERROR, // Processing is encountering errors. 725 ERROR_REC_SRTP_ERROR, // Generic SRTP failure. 726 ERROR_REC_SRTP_AUTH_FAILED, // Failed to authenticate packets. 727 ERROR_REC_TYPING_NOISE_DETECTED, // Typing noise is detected. 728 ERROR_PLAY_DEVICE_OPEN_FAILED = 200, // Could not open playout. 729 ERROR_PLAY_DEVICE_MUTED, // Playout muted by OS. 730 ERROR_PLAY_DEVICE_REMOVED, // Playout removed while active. 731 ERROR_PLAY_RUNTIME_ERROR, // Errors in voice processing. 732 ERROR_PLAY_SRTP_ERROR, // Generic SRTP failure. 733 ERROR_PLAY_SRTP_AUTH_FAILED, // Failed to authenticate packets. 734 ERROR_PLAY_SRTP_REPLAY, // Packet replay detected. 735 }; 736 737 VoiceMediaChannel() {} 738 virtual ~VoiceMediaChannel() {} 739 // Sets the codecs/payload types to be used for incoming media. 740 virtual bool SetRecvCodecs(const std::vector<AudioCodec>& codecs) = 0; 741 // Sets the codecs/payload types to be used for outgoing media. 742 virtual bool SetSendCodecs(const std::vector<AudioCodec>& codecs) = 0; 743 // Starts or stops playout of received audio. 744 virtual bool SetPlayout(bool playout) = 0; 745 // Starts or stops sending (and potentially capture) of local audio. 746 virtual bool SetSend(SendFlags flag) = 0; 747 // Sets the renderer object to be used for the specified remote audio stream. 748 virtual bool SetRemoteRenderer(uint32 ssrc, AudioRenderer* renderer) = 0; 749 // Sets the renderer object to be used for the specified local audio stream. 750 virtual bool SetLocalRenderer(uint32 ssrc, AudioRenderer* renderer) = 0; 751 // Gets current energy levels for all incoming streams. 752 virtual bool GetActiveStreams(AudioInfo::StreamList* actives) = 0; 753 // Get the current energy level of the stream sent to the speaker. 754 virtual int GetOutputLevel() = 0; 755 // Get the time in milliseconds since last recorded keystroke, or negative. 756 virtual int GetTimeSinceLastTyping() = 0; 757 // Temporarily exposed field for tuning typing detect options. 758 virtual void SetTypingDetectionParameters(int time_window, 759 int cost_per_typing, int reporting_threshold, int penalty_decay, 760 int type_event_delay) = 0; 761 // Set left and right scale for speaker output volume of the specified ssrc. 762 virtual bool SetOutputScaling(uint32 ssrc, double left, double right) = 0; 763 // Get left and right scale for speaker output volume of the specified ssrc. 764 virtual bool GetOutputScaling(uint32 ssrc, double* left, double* right) = 0; 765 // Specifies a ringback tone to be played during call setup. 766 virtual bool SetRingbackTone(const char *buf, int len) = 0; 767 // Plays or stops the aforementioned ringback tone 768 virtual bool PlayRingbackTone(uint32 ssrc, bool play, bool loop) = 0; 769 // Returns if the telephone-event has been negotiated. 770 virtual bool CanInsertDtmf() { return false; } 771 // Send and/or play a DTMF |event| according to the |flags|. 772 // The DTMF out-of-band signal will be used on sending. 773 // The |ssrc| should be either 0 or a valid send stream ssrc. 774 // The valid value for the |event| are 0 to 15 which corresponding to 775 // DTMF event 0-9, *, #, A-D. 776 virtual bool InsertDtmf(uint32 ssrc, int event, int duration, int flags) = 0; 777 // Gets quality stats for the channel. 778 virtual bool GetStats(VoiceMediaInfo* info) = 0; 779 // Gets last reported error for this media channel. 780 virtual void GetLastMediaError(uint32* ssrc, 781 VoiceMediaChannel::Error* error) { 782 ASSERT(error != NULL); 783 *error = ERROR_NONE; 784 } 785 // Sets the media options to use. 786 virtual bool SetOptions(const AudioOptions& options) = 0; 787 virtual bool GetOptions(AudioOptions* options) const = 0; 788 789 // Signal errors from MediaChannel. Arguments are: 790 // ssrc(uint32), and error(VoiceMediaChannel::Error). 791 sigslot::signal2<uint32, VoiceMediaChannel::Error> SignalMediaError; 792 }; 793 794 class VideoMediaChannel : public MediaChannel { 795 public: 796 enum Error { 797 ERROR_NONE = 0, // No error. 798 ERROR_OTHER, // Other errors. 799 ERROR_REC_DEVICE_OPEN_FAILED = 100, // Could not open camera. 800 ERROR_REC_DEVICE_NO_DEVICE, // No camera. 801 ERROR_REC_DEVICE_IN_USE, // Device is in already use. 802 ERROR_REC_DEVICE_REMOVED, // Device is removed. 803 ERROR_REC_SRTP_ERROR, // Generic sender SRTP failure. 804 ERROR_REC_SRTP_AUTH_FAILED, // Failed to authenticate packets. 805 ERROR_REC_CPU_MAX_CANT_DOWNGRADE, // Can't downgrade capture anymore. 806 ERROR_PLAY_SRTP_ERROR = 200, // Generic receiver SRTP failure. 807 ERROR_PLAY_SRTP_AUTH_FAILED, // Failed to authenticate packets. 808 ERROR_PLAY_SRTP_REPLAY, // Packet replay detected. 809 }; 810 811 VideoMediaChannel() : renderer_(NULL) {} 812 virtual ~VideoMediaChannel() {} 813 // Sets the codecs/payload types to be used for incoming media. 814 virtual bool SetRecvCodecs(const std::vector<VideoCodec>& codecs) = 0; 815 // Sets the codecs/payload types to be used for outgoing media. 816 virtual bool SetSendCodecs(const std::vector<VideoCodec>& codecs) = 0; 817 // Gets the currently set codecs/payload types to be used for outgoing media. 818 virtual bool GetSendCodec(VideoCodec* send_codec) = 0; 819 // Sets the format of a specified outgoing stream. 820 virtual bool SetSendStreamFormat(uint32 ssrc, const VideoFormat& format) = 0; 821 // Starts or stops playout of received video. 822 virtual bool SetRender(bool render) = 0; 823 // Starts or stops transmission (and potentially capture) of local video. 824 virtual bool SetSend(bool send) = 0; 825 // Sets the renderer object to be used for the specified stream. 826 // If SSRC is 0, the renderer is used for the 'default' stream. 827 virtual bool SetRenderer(uint32 ssrc, VideoRenderer* renderer) = 0; 828 // If |ssrc| is 0, replace the default capturer (engine capturer) with 829 // |capturer|. If |ssrc| is non zero create a new stream with |ssrc| as SSRC. 830 virtual bool SetCapturer(uint32 ssrc, VideoCapturer* capturer) = 0; 831 // Gets quality stats for the channel. 832 virtual bool GetStats(VideoMediaInfo* info) = 0; 833 834 // Send an intra frame to the receivers. 835 virtual bool SendIntraFrame() = 0; 836 // Reuqest each of the remote senders to send an intra frame. 837 virtual bool RequestIntraFrame() = 0; 838 // Sets the media options to use. 839 virtual bool SetOptions(const VideoOptions& options) = 0; 840 virtual bool GetOptions(VideoOptions* options) const = 0; 841 virtual void UpdateAspectRatio(int ratio_w, int ratio_h) = 0; 842 843 // Signal errors from MediaChannel. Arguments are: 844 // ssrc(uint32), and error(VideoMediaChannel::Error). 845 sigslot::signal2<uint32, Error> SignalMediaError; 846 847 protected: 848 VideoRenderer *renderer_; 849 }; 850 851 enum DataMessageType { 852 // TODO(pthatcher): Make this enum match the SCTP PPIDs that WebRTC uses? 853 DMT_CONTROL = 0, 854 DMT_BINARY = 1, 855 DMT_TEXT = 2, 856 }; 857 858 // Info about data received in DataMediaChannel. For use in 859 // DataMediaChannel::SignalDataReceived and in all of the signals that 860 // signal fires, on up the chain. 861 struct ReceiveDataParams { 862 // The in-packet stream indentifier. 863 // For SCTP, this is really SID, not SSRC. 864 uint32 ssrc; 865 // The type of message (binary, text, or control). 866 DataMessageType type; 867 // A per-stream value incremented per packet in the stream. 868 int seq_num; 869 // A per-stream value monotonically increasing with time. 870 int timestamp; 871 872 ReceiveDataParams() : 873 ssrc(0), 874 type(DMT_TEXT), 875 seq_num(0), 876 timestamp(0) { 877 } 878 }; 879 880 struct SendDataParams { 881 // The in-packet stream indentifier. 882 // For SCTP, this is really SID, not SSRC. 883 uint32 ssrc; 884 // The type of message (binary, text, or control). 885 DataMessageType type; 886 887 // For SCTP, whether to send messages flagged as ordered or not. 888 // If false, messages can be received out of order. 889 bool ordered; 890 // For SCTP, whether the messages are sent reliably or not. 891 // If false, messages may be lost. 892 bool reliable; 893 // For SCTP, if reliable == false, provide partial reliability by 894 // resending up to this many times. Either count or millis 895 // is supported, not both at the same time. 896 int max_rtx_count; 897 // For SCTP, if reliable == false, provide partial reliability by 898 // resending for up to this many milliseconds. Either count or millis 899 // is supported, not both at the same time. 900 int max_rtx_ms; 901 902 SendDataParams() : 903 ssrc(0), 904 type(DMT_TEXT), 905 // TODO(pthatcher): Make these true by default? 906 ordered(false), 907 reliable(false), 908 max_rtx_count(0), 909 max_rtx_ms(0) { 910 } 911 }; 912 913 enum SendDataResult { SDR_SUCCESS, SDR_ERROR, SDR_BLOCK }; 914 915 class DataMediaChannel : public MediaChannel { 916 public: 917 enum Error { 918 ERROR_NONE = 0, // No error. 919 ERROR_OTHER, // Other errors. 920 ERROR_SEND_SRTP_ERROR = 200, // Generic SRTP failure. 921 ERROR_SEND_SRTP_AUTH_FAILED, // Failed to authenticate packets. 922 ERROR_RECV_SRTP_ERROR, // Generic SRTP failure. 923 ERROR_RECV_SRTP_AUTH_FAILED, // Failed to authenticate packets. 924 ERROR_RECV_SRTP_REPLAY, // Packet replay detected. 925 }; 926 927 virtual ~DataMediaChannel() {} 928 929 virtual bool SetSendBandwidth(bool autobw, int bps) = 0; 930 virtual bool SetSendCodecs(const std::vector<DataCodec>& codecs) = 0; 931 virtual bool SetRecvCodecs(const std::vector<DataCodec>& codecs) = 0; 932 virtual bool SetRecvRtpHeaderExtensions( 933 const std::vector<RtpHeaderExtension>& extensions) = 0; 934 virtual bool SetSendRtpHeaderExtensions( 935 const std::vector<RtpHeaderExtension>& extensions) = 0; 936 virtual bool AddSendStream(const StreamParams& sp) = 0; 937 virtual bool RemoveSendStream(uint32 ssrc) = 0; 938 virtual bool AddRecvStream(const StreamParams& sp) = 0; 939 virtual bool RemoveRecvStream(uint32 ssrc) = 0; 940 virtual bool MuteStream(uint32 ssrc, bool on) { return false; } 941 // TODO(pthatcher): Implement this. 942 virtual bool GetStats(DataMediaInfo* info) { return true; } 943 944 virtual bool SetSend(bool send) = 0; 945 virtual bool SetReceive(bool receive) = 0; 946 virtual void OnPacketReceived(talk_base::Buffer* packet) = 0; 947 virtual void OnRtcpReceived(talk_base::Buffer* packet) = 0; 948 949 virtual bool SendData( 950 const SendDataParams& params, 951 const talk_base::Buffer& payload, 952 SendDataResult* result = NULL) = 0; 953 // Signals when data is received (params, data, len) 954 sigslot::signal3<const ReceiveDataParams&, 955 const char*, 956 size_t> SignalDataReceived; 957 // Signal errors from MediaChannel. Arguments are: 958 // ssrc(uint32), and error(DataMediaChannel::Error). 959 sigslot::signal2<uint32, DataMediaChannel::Error> SignalMediaError; 960 // Signal when the media channel is ready to send the stream. Arguments are: 961 // writable(bool) 962 sigslot::signal1<bool> SignalReadyToSend; 963 }; 964 965 } // namespace cricket 966 967 #endif // TALK_MEDIA_BASE_MEDIACHANNEL_H_ 968
