1 /* 2 * libjingle 3 * Copyright 2004 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #ifndef TALK_MEDIA_WEBRTCVOICEENGINE_H_ 29 #define TALK_MEDIA_WEBRTCVOICEENGINE_H_ 30 31 #include <map> 32 #include <set> 33 #include <string> 34 #include <vector> 35 36 #include "talk/base/buffer.h" 37 #include "talk/base/byteorder.h" 38 #include "talk/base/logging.h" 39 #include "talk/base/scoped_ptr.h" 40 #include "talk/base/stream.h" 41 #include "talk/media/base/rtputils.h" 42 #include "talk/media/webrtc/webrtccommon.h" 43 #include "talk/media/webrtc/webrtcexport.h" 44 #include "talk/media/webrtc/webrtcvoe.h" 45 #include "talk/session/media/channel.h" 46 #include "webrtc/common.h" 47 #include "webrtc/modules/audio_coding/main/interface/audio_coding_module.h" 48 49 #if !defined(LIBPEERCONNECTION_LIB) && \ 50 !defined(LIBPEERCONNECTION_IMPLEMENTATION) 51 #error "Bogus include." 52 #endif 53 54 55 namespace cricket { 56 57 // WebRtcSoundclipStream is an adapter object that allows a memory stream to be 58 // passed into WebRtc, and support looping. 59 class WebRtcSoundclipStream : public webrtc::InStream { 60 public: 61 WebRtcSoundclipStream(const char* buf, size_t len) 62 : mem_(buf, len), loop_(true) { 63 } 64 void set_loop(bool loop) { loop_ = loop; } 65 virtual int Read(void* buf, int len); 66 virtual int Rewind(); 67 68 private: 69 talk_base::MemoryStream mem_; 70 bool loop_; 71 }; 72 73 // WebRtcMonitorStream is used to monitor a stream coming from WebRtc. 74 // For now we just dump the data. 75 class WebRtcMonitorStream : public webrtc::OutStream { 76 virtual bool Write(const void *buf, int len) { 77 return true; 78 } 79 }; 80 81 class AudioDeviceModule; 82 class AudioRenderer; 83 class VoETraceWrapper; 84 class VoEWrapper; 85 class VoiceProcessor; 86 class WebRtcSoundclipMedia; 87 class WebRtcVoiceMediaChannel; 88 89 // WebRtcVoiceEngine is a class to be used with CompositeMediaEngine. 90 // It uses the WebRtc VoiceEngine library for audio handling. 91 class WebRtcVoiceEngine 92 : public webrtc::VoiceEngineObserver, 93 public webrtc::TraceCallback, 94 public webrtc::VoEMediaProcess { 95 public: 96 WebRtcVoiceEngine(); 97 // Dependency injection for testing. 98 WebRtcVoiceEngine(VoEWrapper* voe_wrapper, 99 VoEWrapper* voe_wrapper_sc, 100 VoETraceWrapper* tracing); 101 ~WebRtcVoiceEngine(); 102 bool Init(talk_base::Thread* worker_thread); 103 void Terminate(); 104 105 int GetCapabilities(); 106 VoiceMediaChannel* CreateChannel(); 107 108 SoundclipMedia* CreateSoundclip(); 109 110 AudioOptions GetOptions() const { return options_; } 111 bool SetOptions(const AudioOptions& options); 112 // Overrides, when set, take precedence over the options on a 113 // per-option basis. For example, if AGC is set in options and AEC 114 // is set in overrides, AGC and AEC will be both be set. Overrides 115 // can also turn off options. For example, if AGC is set to "on" in 116 // options and AGC is set to "off" in overrides, the result is that 117 // AGC will be off until different overrides are applied or until 118 // the overrides are cleared. Only one set of overrides is present 119 // at a time (they do not "stack"). And when the overrides are 120 // cleared, the media engine's state reverts back to the options set 121 // via SetOptions. This allows us to have both "persistent options" 122 // (the normal options) and "temporary options" (overrides). 123 bool SetOptionOverrides(const AudioOptions& options); 124 bool ClearOptionOverrides(); 125 bool SetDelayOffset(int offset); 126 bool SetDevices(const Device* in_device, const Device* out_device); 127 bool GetOutputVolume(int* level); 128 bool SetOutputVolume(int level); 129 int GetInputLevel(); 130 bool SetLocalMonitor(bool enable); 131 132 const std::vector<AudioCodec>& codecs(); 133 bool FindCodec(const AudioCodec& codec); 134 bool FindWebRtcCodec(const AudioCodec& codec, webrtc::CodecInst* gcodec); 135 136 const std::vector<RtpHeaderExtension>& rtp_header_extensions() const; 137 138 void SetLogging(int min_sev, const char* filter); 139 140 bool RegisterProcessor(uint32 ssrc, 141 VoiceProcessor* voice_processor, 142 MediaProcessorDirection direction); 143 bool UnregisterProcessor(uint32 ssrc, 144 VoiceProcessor* voice_processor, 145 MediaProcessorDirection direction); 146 147 // Method from webrtc::VoEMediaProcess 148 virtual void Process(int channel, 149 webrtc::ProcessingTypes type, 150 int16_t audio10ms[], 151 int length, 152 int sampling_freq, 153 bool is_stereo); 154 155 // For tracking WebRtc channels. Needed because we have to pause them 156 // all when switching devices. 157 // May only be called by WebRtcVoiceMediaChannel. 158 void RegisterChannel(WebRtcVoiceMediaChannel *channel); 159 void UnregisterChannel(WebRtcVoiceMediaChannel *channel); 160 161 // May only be called by WebRtcSoundclipMedia. 162 void RegisterSoundclip(WebRtcSoundclipMedia *channel); 163 void UnregisterSoundclip(WebRtcSoundclipMedia *channel); 164 165 // Called by WebRtcVoiceMediaChannel to set a gain offset from 166 // the default AGC target level. 167 bool AdjustAgcLevel(int delta); 168 169 VoEWrapper* voe() { return voe_wrapper_.get(); } 170 VoEWrapper* voe_sc() { return voe_wrapper_sc_.get(); } 171 int GetLastEngineError(); 172 173 // Set the external ADMs. This can only be called before Init. 174 bool SetAudioDeviceModule(webrtc::AudioDeviceModule* adm, 175 webrtc::AudioDeviceModule* adm_sc); 176 177 // Check whether the supplied trace should be ignored. 178 bool ShouldIgnoreTrace(const std::string& trace); 179 180 // Create a VoiceEngine Channel. 181 int CreateMediaVoiceChannel(); 182 int CreateSoundclipVoiceChannel(); 183 184 private: 185 typedef std::vector<WebRtcSoundclipMedia *> SoundclipList; 186 typedef std::vector<WebRtcVoiceMediaChannel *> ChannelList; 187 typedef sigslot:: 188 signal3<uint32, MediaProcessorDirection, AudioFrame*> FrameSignal; 189 190 void Construct(); 191 void ConstructCodecs(); 192 bool InitInternal(); 193 bool EnsureSoundclipEngineInit(); 194 void SetTraceFilter(int filter); 195 void SetTraceOptions(const std::string& options); 196 // Applies either options or overrides. Every option that is "set" 197 // will be applied. Every option not "set" will be ignored. This 198 // allows us to selectively turn on and off different options easily 199 // at any time. 200 bool ApplyOptions(const AudioOptions& options); 201 // Configure for using ACM2, if |enable| is true, otherwise configure for 202 // ACM1. 203 void EnableExperimentalAcm(bool enable); 204 virtual void Print(webrtc::TraceLevel level, const char* trace, int length); 205 virtual void CallbackOnError(int channel, int errCode); 206 // Given the device type, name, and id, find device id. Return true and 207 // set the output parameter rtc_id if successful. 208 bool FindWebRtcAudioDeviceId( 209 bool is_input, const std::string& dev_name, int dev_id, int* rtc_id); 210 bool FindChannelAndSsrc(int channel_num, 211 WebRtcVoiceMediaChannel** channel, 212 uint32* ssrc) const; 213 bool FindChannelNumFromSsrc(uint32 ssrc, 214 MediaProcessorDirection direction, 215 int* channel_num); 216 bool ChangeLocalMonitor(bool enable); 217 bool PauseLocalMonitor(); 218 bool ResumeLocalMonitor(); 219 220 bool UnregisterProcessorChannel(MediaProcessorDirection channel_direction, 221 uint32 ssrc, 222 VoiceProcessor* voice_processor, 223 MediaProcessorDirection processor_direction); 224 225 void StartAecDump(const std::string& filename); 226 void StopAecDump(); 227 int CreateVoiceChannel(VoEWrapper* voe); 228 229 // When a voice processor registers with the engine, it is connected 230 // to either the Rx or Tx signals, based on the direction parameter. 231 // SignalXXMediaFrame will be invoked for every audio packet. 232 FrameSignal SignalRxMediaFrame; 233 FrameSignal SignalTxMediaFrame; 234 235 static const int kDefaultLogSeverity = talk_base::LS_WARNING; 236 237 // The primary instance of WebRtc VoiceEngine. 238 talk_base::scoped_ptr<VoEWrapper> voe_wrapper_; 239 // A secondary instance, for playing out soundclips (on the 'ring' device). 240 talk_base::scoped_ptr<VoEWrapper> voe_wrapper_sc_; 241 bool voe_wrapper_sc_initialized_; 242 talk_base::scoped_ptr<VoETraceWrapper> tracing_; 243 // The external audio device manager 244 webrtc::AudioDeviceModule* adm_; 245 webrtc::AudioDeviceModule* adm_sc_; 246 int log_filter_; 247 std::string log_options_; 248 bool is_dumping_aec_; 249 std::vector<AudioCodec> codecs_; 250 std::vector<RtpHeaderExtension> rtp_header_extensions_; 251 bool desired_local_monitor_enable_; 252 talk_base::scoped_ptr<WebRtcMonitorStream> monitor_; 253 SoundclipList soundclips_; 254 ChannelList channels_; 255 // channels_ can be read from WebRtc callback thread. We need a lock on that 256 // callback as well as the RegisterChannel/UnregisterChannel. 257 talk_base::CriticalSection channels_cs_; 258 webrtc::AgcConfig default_agc_config_; 259 260 webrtc::Config voe_config_; 261 bool use_experimental_acm_; 262 263 bool initialized_; 264 // See SetOptions and SetOptionOverrides for a description of the 265 // difference between options and overrides. 266 // options_ are the base options, which combined with the 267 // option_overrides_, create the current options being used. 268 // options_ is stored so that when option_overrides_ is cleared, we 269 // can restore the options_ without the option_overrides. 270 AudioOptions options_; 271 AudioOptions option_overrides_; 272 273 // When the media processor registers with the engine, the ssrc is cached 274 // here so that a look up need not be made when the callback is invoked. 275 // This is necessary because the lookup results in mux_channels_cs lock being 276 // held and if a remote participant leaves the hangout at the same time 277 // we hit a deadlock. 278 uint32 tx_processor_ssrc_; 279 uint32 rx_processor_ssrc_; 280 281 talk_base::CriticalSection signal_media_critical_; 282 }; 283 284 // WebRtcMediaChannel is a class that implements the common WebRtc channel 285 // functionality. 286 template <class T, class E> 287 class WebRtcMediaChannel : public T, public webrtc::Transport { 288 public: 289 WebRtcMediaChannel(E *engine, int channel) 290 : engine_(engine), voe_channel_(channel) {} 291 E *engine() { return engine_; } 292 int voe_channel() const { return voe_channel_; } 293 bool valid() const { return voe_channel_ != -1; } 294 295 protected: 296 // implements Transport interface 297 virtual int SendPacket(int channel, const void *data, int len) { 298 talk_base::Buffer packet(data, len, kMaxRtpPacketLen); 299 if (!T::SendPacket(&packet)) { 300 return -1; 301 } 302 return len; 303 } 304 305 virtual int SendRTCPPacket(int channel, const void *data, int len) { 306 talk_base::Buffer packet(data, len, kMaxRtpPacketLen); 307 return T::SendRtcp(&packet) ? len : -1; 308 } 309 310 private: 311 E *engine_; 312 int voe_channel_; 313 }; 314 315 // WebRtcVoiceMediaChannel is an implementation of VoiceMediaChannel that uses 316 // WebRtc Voice Engine. 317 class WebRtcVoiceMediaChannel 318 : public WebRtcMediaChannel<VoiceMediaChannel, WebRtcVoiceEngine> { 319 public: 320 explicit WebRtcVoiceMediaChannel(WebRtcVoiceEngine *engine); 321 virtual ~WebRtcVoiceMediaChannel(); 322 virtual bool SetOptions(const AudioOptions& options); 323 virtual bool GetOptions(AudioOptions* options) const { 324 *options = options_; 325 return true; 326 } 327 virtual bool SetRecvCodecs(const std::vector<AudioCodec> &codecs); 328 virtual bool SetSendCodecs(const std::vector<AudioCodec> &codecs); 329 virtual bool SetRecvRtpHeaderExtensions( 330 const std::vector<RtpHeaderExtension>& extensions); 331 virtual bool SetSendRtpHeaderExtensions( 332 const std::vector<RtpHeaderExtension>& extensions); 333 virtual bool SetPlayout(bool playout); 334 bool PausePlayout(); 335 bool ResumePlayout(); 336 virtual bool SetSend(SendFlags send); 337 bool PauseSend(); 338 bool ResumeSend(); 339 virtual bool AddSendStream(const StreamParams& sp); 340 virtual bool RemoveSendStream(uint32 ssrc); 341 virtual bool AddRecvStream(const StreamParams& sp); 342 virtual bool RemoveRecvStream(uint32 ssrc); 343 virtual bool SetRemoteRenderer(uint32 ssrc, AudioRenderer* renderer); 344 virtual bool SetLocalRenderer(uint32 ssrc, AudioRenderer* renderer); 345 virtual bool GetActiveStreams(AudioInfo::StreamList* actives); 346 virtual int GetOutputLevel(); 347 virtual int GetTimeSinceLastTyping(); 348 virtual void SetTypingDetectionParameters(int time_window, 349 int cost_per_typing, int reporting_threshold, int penalty_decay, 350 int type_event_delay); 351 virtual bool SetOutputScaling(uint32 ssrc, double left, double right); 352 virtual bool GetOutputScaling(uint32 ssrc, double* left, double* right); 353 354 virtual bool SetRingbackTone(const char *buf, int len); 355 virtual bool PlayRingbackTone(uint32 ssrc, bool play, bool loop); 356 virtual bool CanInsertDtmf(); 357 virtual bool InsertDtmf(uint32 ssrc, int event, int duration, int flags); 358 359 virtual void OnPacketReceived(talk_base::Buffer* packet, 360 const talk_base::PacketTime& packet_time); 361 virtual void OnRtcpReceived(talk_base::Buffer* packet, 362 const talk_base::PacketTime& packet_time); 363 virtual void OnReadyToSend(bool ready) {} 364 virtual bool MuteStream(uint32 ssrc, bool on); 365 virtual bool SetSendBandwidth(bool autobw, int bps); 366 virtual bool GetStats(VoiceMediaInfo* info); 367 // Gets last reported error from WebRtc voice engine. This should be only 368 // called in response a failure. 369 virtual void GetLastMediaError(uint32* ssrc, 370 VoiceMediaChannel::Error* error); 371 bool FindSsrc(int channel_num, uint32* ssrc); 372 void OnError(uint32 ssrc, int error); 373 374 bool sending() const { return send_ != SEND_NOTHING; } 375 int GetReceiveChannelNum(uint32 ssrc); 376 int GetSendChannelNum(uint32 ssrc); 377 378 protected: 379 int GetLastEngineError() { return engine()->GetLastEngineError(); } 380 int GetOutputLevel(int channel); 381 bool GetRedSendCodec(const AudioCodec& red_codec, 382 const std::vector<AudioCodec>& all_codecs, 383 webrtc::CodecInst* send_codec); 384 bool EnableRtcp(int channel); 385 bool ResetRecvCodecs(int channel); 386 bool SetPlayout(int channel, bool playout); 387 static uint32 ParseSsrc(const void* data, size_t len, bool rtcp); 388 static Error WebRtcErrorToChannelError(int err_code); 389 390 private: 391 struct WebRtcVoiceChannelInfo; 392 typedef std::map<uint32, WebRtcVoiceChannelInfo> ChannelMap; 393 394 void SetNack(int channel, bool nack_enabled); 395 void SetNack(const ChannelMap& channels, bool nack_enabled); 396 bool SetSendCodec(const webrtc::CodecInst& send_codec); 397 bool SetSendCodec(int channel, const webrtc::CodecInst& send_codec); 398 bool ChangePlayout(bool playout); 399 bool ChangeSend(SendFlags send); 400 bool ChangeSend(int channel, SendFlags send); 401 void ConfigureSendChannel(int channel); 402 bool ConfigureRecvChannel(int channel); 403 bool DeleteChannel(int channel); 404 bool InConferenceMode() const { 405 return options_.conference_mode.GetWithDefaultIfUnset(false); 406 } 407 bool IsDefaultChannel(int channel_id) const { 408 return channel_id == voe_channel(); 409 } 410 bool SetSendCodecs(int channel, const std::vector<AudioCodec>& codecs); 411 bool SetSendBandwidthInternal(bool autobw, int bps); 412 413 talk_base::scoped_ptr<WebRtcSoundclipStream> ringback_tone_; 414 std::set<int> ringback_channels_; // channels playing ringback 415 std::vector<AudioCodec> recv_codecs_; 416 std::vector<AudioCodec> send_codecs_; 417 talk_base::scoped_ptr<webrtc::CodecInst> send_codec_; 418 bool send_bw_setting_; 419 bool send_autobw_; 420 int send_bw_bps_; 421 AudioOptions options_; 422 bool dtmf_allowed_; 423 bool desired_playout_; 424 bool nack_enabled_; 425 bool playout_; 426 bool typing_noise_detected_; 427 SendFlags desired_send_; 428 SendFlags send_; 429 430 // send_channels_ contains the channels which are being used for sending. 431 // When the default channel (voe_channel) is used for sending, it is 432 // contained in send_channels_, otherwise not. 433 ChannelMap send_channels_; 434 uint32 default_receive_ssrc_; 435 // Note the default channel (voe_channel()) can reside in both 436 // receive_channels_ and send_channels_ in non-conference mode and in that 437 // case it will only be there if a non-zero default_receive_ssrc_ is set. 438 ChannelMap receive_channels_; // for multiple sources 439 // receive_channels_ can be read from WebRtc callback thread. Access from 440 // the WebRtc thread must be synchronized with edits on the worker thread. 441 // Reads on the worker thread are ok. 442 // 443 // Do not lock this on the VoE media processor thread; potential for deadlock 444 // exists. 445 mutable talk_base::CriticalSection receive_channels_cs_; 446 }; 447 448 } // namespace cricket 449 450 #endif // TALK_MEDIA_WEBRTCVOICEENGINE_H_ 451
