1 /* 2 * libjingle 3 * Copyright 2004 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #ifndef TALK_MEDIA_WEBRTCVOICEENGINE_H_ 29 #define TALK_MEDIA_WEBRTCVOICEENGINE_H_ 30 31 #include <map> 32 #include <set> 33 #include <string> 34 #include <vector> 35 36 #include "talk/base/buffer.h" 37 #include "talk/base/byteorder.h" 38 #include "talk/base/logging.h" 39 #include "talk/base/scoped_ptr.h" 40 #include "talk/base/stream.h" 41 #include "talk/media/base/rtputils.h" 42 #include "talk/media/webrtc/webrtccommon.h" 43 #include "talk/media/webrtc/webrtcexport.h" 44 #include "talk/media/webrtc/webrtcvoe.h" 45 #include "talk/session/media/channel.h" 46 47 #if !defined(LIBPEERCONNECTION_LIB) && \ 48 !defined(LIBPEERCONNECTION_IMPLEMENTATION) 49 #error "Bogus include." 50 #endif 51 52 53 namespace cricket { 54 55 // WebRtcSoundclipStream is an adapter object that allows a memory stream to be 56 // passed into WebRtc, and support looping. 57 class WebRtcSoundclipStream : public webrtc::InStream { 58 public: 59 WebRtcSoundclipStream(const char* buf, size_t len) 60 : mem_(buf, len), loop_(true) { 61 } 62 void set_loop(bool loop) { loop_ = loop; } 63 virtual int Read(void* buf, int len); 64 virtual int Rewind(); 65 66 private: 67 talk_base::MemoryStream mem_; 68 bool loop_; 69 }; 70 71 // WebRtcMonitorStream is used to monitor a stream coming from WebRtc. 72 // For now we just dump the data. 73 class WebRtcMonitorStream : public webrtc::OutStream { 74 virtual bool Write(const void *buf, int len) { 75 return true; 76 } 77 }; 78 79 class AudioDeviceModule; 80 class AudioRenderer; 81 class VoETraceWrapper; 82 class VoEWrapper; 83 class VoiceProcessor; 84 class WebRtcSoundclipMedia; 85 class WebRtcVoiceMediaChannel; 86 87 // WebRtcVoiceEngine is a class to be used with CompositeMediaEngine. 88 // It uses the WebRtc VoiceEngine library for audio handling. 89 class WebRtcVoiceEngine 90 : public webrtc::VoiceEngineObserver, 91 public webrtc::TraceCallback, 92 public webrtc::VoEMediaProcess { 93 public: 94 WebRtcVoiceEngine(); 95 // Dependency injection for testing. 96 WebRtcVoiceEngine(VoEWrapper* voe_wrapper, 97 VoEWrapper* voe_wrapper_sc, 98 VoETraceWrapper* tracing); 99 ~WebRtcVoiceEngine(); 100 bool Init(talk_base::Thread* worker_thread); 101 void Terminate(); 102 103 int GetCapabilities(); 104 VoiceMediaChannel* CreateChannel(); 105 106 SoundclipMedia* CreateSoundclip(); 107 108 // TODO(pthatcher): Rename to SetOptions and replace the old 109 // flags-based SetOptions. 110 bool SetAudioOptions(const AudioOptions& options); 111 // Eventually, we will replace them with AudioOptions. 112 // In the meantime, we leave this here for backwards compat. 113 bool SetOptions(int flags); 114 // Overrides, when set, take precedence over the options on a 115 // per-option basis. For example, if AGC is set in options and AEC 116 // is set in overrides, AGC and AEC will be both be set. Overrides 117 // can also turn off options. For example, if AGC is set to "on" in 118 // options and AGC is set to "off" in overrides, the result is that 119 // AGC will be off until different overrides are applied or until 120 // the overrides are cleared. Only one set of overrides is present 121 // at a time (they do not "stack"). And when the overrides are 122 // cleared, the media engine's state reverts back to the options set 123 // via SetOptions. This allows us to have both "persistent options" 124 // (the normal options) and "temporary options" (overrides). 125 bool SetOptionOverrides(const AudioOptions& options); 126 bool ClearOptionOverrides(); 127 bool SetDelayOffset(int offset); 128 bool SetDevices(const Device* in_device, const Device* out_device); 129 bool GetOutputVolume(int* level); 130 bool SetOutputVolume(int level); 131 int GetInputLevel(); 132 bool SetLocalMonitor(bool enable); 133 134 const std::vector<AudioCodec>& codecs(); 135 bool FindCodec(const AudioCodec& codec); 136 bool FindWebRtcCodec(const AudioCodec& codec, webrtc::CodecInst* gcodec); 137 138 const std::vector<RtpHeaderExtension>& rtp_header_extensions() const; 139 140 void SetLogging(int min_sev, const char* filter); 141 142 bool RegisterProcessor(uint32 ssrc, 143 VoiceProcessor* voice_processor, 144 MediaProcessorDirection direction); 145 bool UnregisterProcessor(uint32 ssrc, 146 VoiceProcessor* voice_processor, 147 MediaProcessorDirection direction); 148 149 // Method from webrtc::VoEMediaProcess 150 virtual void Process(int channel, 151 webrtc::ProcessingTypes type, 152 int16_t audio10ms[], 153 int length, 154 int sampling_freq, 155 bool is_stereo); 156 157 // For tracking WebRtc channels. Needed because we have to pause them 158 // all when switching devices. 159 // May only be called by WebRtcVoiceMediaChannel. 160 void RegisterChannel(WebRtcVoiceMediaChannel *channel); 161 void UnregisterChannel(WebRtcVoiceMediaChannel *channel); 162 163 // May only be called by WebRtcSoundclipMedia. 164 void RegisterSoundclip(WebRtcSoundclipMedia *channel); 165 void UnregisterSoundclip(WebRtcSoundclipMedia *channel); 166 167 // Called by WebRtcVoiceMediaChannel to set a gain offset from 168 // the default AGC target level. 169 bool AdjustAgcLevel(int delta); 170 171 VoEWrapper* voe() { return voe_wrapper_.get(); } 172 VoEWrapper* voe_sc() { return voe_wrapper_sc_.get(); } 173 int GetLastEngineError(); 174 175 // Set the external ADMs. This can only be called before Init. 176 bool SetAudioDeviceModule(webrtc::AudioDeviceModule* adm, 177 webrtc::AudioDeviceModule* adm_sc); 178 179 // Check whether the supplied trace should be ignored. 180 bool ShouldIgnoreTrace(const std::string& trace); 181 182 private: 183 typedef std::vector<WebRtcSoundclipMedia *> SoundclipList; 184 typedef std::vector<WebRtcVoiceMediaChannel *> ChannelList; 185 typedef sigslot:: 186 signal3<uint32, MediaProcessorDirection, AudioFrame*> FrameSignal; 187 188 void Construct(); 189 void ConstructCodecs(); 190 bool InitInternal(); 191 void SetTraceFilter(int filter); 192 void SetTraceOptions(const std::string& options); 193 // Applies either options or overrides. Every option that is "set" 194 // will be applied. Every option not "set" will be ignored. This 195 // allows us to selectively turn on and off different options easily 196 // at any time. 197 bool ApplyOptions(const AudioOptions& options); 198 virtual void Print(webrtc::TraceLevel level, const char* trace, int length); 199 virtual void CallbackOnError(int channel, int errCode); 200 // Given the device type, name, and id, find device id. Return true and 201 // set the output parameter rtc_id if successful. 202 bool FindWebRtcAudioDeviceId( 203 bool is_input, const std::string& dev_name, int dev_id, int* rtc_id); 204 bool FindChannelAndSsrc(int channel_num, 205 WebRtcVoiceMediaChannel** channel, 206 uint32* ssrc) const; 207 bool FindChannelNumFromSsrc(uint32 ssrc, 208 MediaProcessorDirection direction, 209 int* channel_num); 210 bool ChangeLocalMonitor(bool enable); 211 bool PauseLocalMonitor(); 212 bool ResumeLocalMonitor(); 213 214 bool UnregisterProcessorChannel(MediaProcessorDirection channel_direction, 215 uint32 ssrc, 216 VoiceProcessor* voice_processor, 217 MediaProcessorDirection processor_direction); 218 219 void StartAecDump(const std::string& filename); 220 void StopAecDump(); 221 222 // When a voice processor registers with the engine, it is connected 223 // to either the Rx or Tx signals, based on the direction parameter. 224 // SignalXXMediaFrame will be invoked for every audio packet. 225 FrameSignal SignalRxMediaFrame; 226 FrameSignal SignalTxMediaFrame; 227 228 static const int kDefaultLogSeverity = talk_base::LS_WARNING; 229 230 // The primary instance of WebRtc VoiceEngine. 231 talk_base::scoped_ptr<VoEWrapper> voe_wrapper_; 232 // A secondary instance, for playing out soundclips (on the 'ring' device). 233 talk_base::scoped_ptr<VoEWrapper> voe_wrapper_sc_; 234 talk_base::scoped_ptr<VoETraceWrapper> tracing_; 235 // The external audio device manager 236 webrtc::AudioDeviceModule* adm_; 237 webrtc::AudioDeviceModule* adm_sc_; 238 int log_filter_; 239 std::string log_options_; 240 bool is_dumping_aec_; 241 std::vector<AudioCodec> codecs_; 242 std::vector<RtpHeaderExtension> rtp_header_extensions_; 243 bool desired_local_monitor_enable_; 244 talk_base::scoped_ptr<WebRtcMonitorStream> monitor_; 245 SoundclipList soundclips_; 246 ChannelList channels_; 247 // channels_ can be read from WebRtc callback thread. We need a lock on that 248 // callback as well as the RegisterChannel/UnregisterChannel. 249 talk_base::CriticalSection channels_cs_; 250 webrtc::AgcConfig default_agc_config_; 251 bool initialized_; 252 // See SetOptions and SetOptionOverrides for a description of the 253 // difference between options and overrides. 254 // options_ are the base options, which combined with the 255 // option_overrides_, create the current options being used. 256 // options_ is stored so that when option_overrides_ is cleared, we 257 // can restore the options_ without the option_overrides. 258 AudioOptions options_; 259 AudioOptions option_overrides_; 260 261 // When the media processor registers with the engine, the ssrc is cached 262 // here so that a look up need not be made when the callback is invoked. 263 // This is necessary because the lookup results in mux_channels_cs lock being 264 // held and if a remote participant leaves the hangout at the same time 265 // we hit a deadlock. 266 uint32 tx_processor_ssrc_; 267 uint32 rx_processor_ssrc_; 268 269 talk_base::CriticalSection signal_media_critical_; 270 }; 271 272 // WebRtcMediaChannel is a class that implements the common WebRtc channel 273 // functionality. 274 template <class T, class E> 275 class WebRtcMediaChannel : public T, public webrtc::Transport { 276 public: 277 WebRtcMediaChannel(E *engine, int channel) 278 : engine_(engine), voe_channel_(channel) {} 279 E *engine() { return engine_; } 280 int voe_channel() const { return voe_channel_; } 281 bool valid() const { return voe_channel_ != -1; } 282 283 protected: 284 // implements Transport interface 285 virtual int SendPacket(int channel, const void *data, int len) { 286 talk_base::Buffer packet(data, len, kMaxRtpPacketLen); 287 if (!T::SendPacket(&packet)) { 288 return -1; 289 } 290 return len; 291 } 292 293 virtual int SendRTCPPacket(int channel, const void *data, int len) { 294 talk_base::Buffer packet(data, len, kMaxRtpPacketLen); 295 return T::SendRtcp(&packet) ? len : -1; 296 } 297 298 private: 299 E *engine_; 300 int voe_channel_; 301 }; 302 303 // WebRtcVoiceMediaChannel is an implementation of VoiceMediaChannel that uses 304 // WebRtc Voice Engine. 305 class WebRtcVoiceMediaChannel 306 : public WebRtcMediaChannel<VoiceMediaChannel, WebRtcVoiceEngine> { 307 public: 308 explicit WebRtcVoiceMediaChannel(WebRtcVoiceEngine *engine); 309 virtual ~WebRtcVoiceMediaChannel(); 310 virtual bool SetOptions(const AudioOptions& options); 311 virtual bool GetOptions(AudioOptions* options) const { 312 *options = options_; 313 return true; 314 } 315 virtual bool SetRecvCodecs(const std::vector<AudioCodec> &codecs); 316 virtual bool SetSendCodecs(const std::vector<AudioCodec> &codecs); 317 virtual bool SetRecvRtpHeaderExtensions( 318 const std::vector<RtpHeaderExtension>& extensions); 319 virtual bool SetSendRtpHeaderExtensions( 320 const std::vector<RtpHeaderExtension>& extensions); 321 virtual bool SetPlayout(bool playout); 322 bool PausePlayout(); 323 bool ResumePlayout(); 324 virtual bool SetSend(SendFlags send); 325 bool PauseSend(); 326 bool ResumeSend(); 327 virtual bool AddSendStream(const StreamParams& sp); 328 virtual bool RemoveSendStream(uint32 ssrc); 329 virtual bool AddRecvStream(const StreamParams& sp); 330 virtual bool RemoveRecvStream(uint32 ssrc); 331 virtual bool SetRemoteRenderer(uint32 ssrc, AudioRenderer* renderer); 332 virtual bool SetLocalRenderer(uint32 ssrc, AudioRenderer* renderer); 333 virtual bool GetActiveStreams(AudioInfo::StreamList* actives); 334 virtual int GetOutputLevel(); 335 virtual int GetTimeSinceLastTyping(); 336 virtual void SetTypingDetectionParameters(int time_window, 337 int cost_per_typing, int reporting_threshold, int penalty_decay, 338 int type_event_delay); 339 virtual bool SetOutputScaling(uint32 ssrc, double left, double right); 340 virtual bool GetOutputScaling(uint32 ssrc, double* left, double* right); 341 342 virtual bool SetRingbackTone(const char *buf, int len); 343 virtual bool PlayRingbackTone(uint32 ssrc, bool play, bool loop); 344 virtual bool CanInsertDtmf(); 345 virtual bool InsertDtmf(uint32 ssrc, int event, int duration, int flags); 346 347 virtual void OnPacketReceived(talk_base::Buffer* packet); 348 virtual void OnRtcpReceived(talk_base::Buffer* packet); 349 virtual void OnReadyToSend(bool ready) {} 350 virtual bool MuteStream(uint32 ssrc, bool on); 351 virtual bool SetSendBandwidth(bool autobw, int bps); 352 virtual bool GetStats(VoiceMediaInfo* info); 353 // Gets last reported error from WebRtc voice engine. This should be only 354 // called in response a failure. 355 virtual void GetLastMediaError(uint32* ssrc, 356 VoiceMediaChannel::Error* error); 357 bool FindSsrc(int channel_num, uint32* ssrc); 358 void OnError(uint32 ssrc, int error); 359 360 bool sending() const { return send_ != SEND_NOTHING; } 361 int GetReceiveChannelNum(uint32 ssrc); 362 int GetSendChannelNum(uint32 ssrc); 363 364 protected: 365 int GetLastEngineError() { return engine()->GetLastEngineError(); } 366 int GetOutputLevel(int channel); 367 bool GetRedSendCodec(const AudioCodec& red_codec, 368 const std::vector<AudioCodec>& all_codecs, 369 webrtc::CodecInst* send_codec); 370 bool EnableRtcp(int channel); 371 bool ResetRecvCodecs(int channel); 372 bool SetPlayout(int channel, bool playout); 373 static uint32 ParseSsrc(const void* data, size_t len, bool rtcp); 374 static Error WebRtcErrorToChannelError(int err_code); 375 376 private: 377 struct WebRtcVoiceChannelInfo; 378 typedef std::map<uint32, WebRtcVoiceChannelInfo> ChannelMap; 379 380 void SetNack(uint32 ssrc, int channel, bool nack_enabled); 381 void SetNack(const ChannelMap& channels, bool nack_enabled); 382 bool SetSendCodec(const webrtc::CodecInst& send_codec); 383 bool SetSendCodec(int channel, const webrtc::CodecInst& send_codec); 384 bool ChangePlayout(bool playout); 385 bool ChangeSend(SendFlags send); 386 bool ChangeSend(int channel, SendFlags send); 387 void ConfigureSendChannel(int channel); 388 bool DeleteChannel(int channel); 389 bool InConferenceMode() const { 390 return options_.conference_mode.GetWithDefaultIfUnset(false); 391 } 392 bool IsDefaultChannel(int channel_id) const { 393 return channel_id == voe_channel(); 394 } 395 396 talk_base::scoped_ptr<WebRtcSoundclipStream> ringback_tone_; 397 std::set<int> ringback_channels_; // channels playing ringback 398 std::vector<AudioCodec> recv_codecs_; 399 talk_base::scoped_ptr<webrtc::CodecInst> send_codec_; 400 AudioOptions options_; 401 bool dtmf_allowed_; 402 bool desired_playout_; 403 bool nack_enabled_; 404 bool playout_; 405 SendFlags desired_send_; 406 SendFlags send_; 407 408 // send_channels_ contains the channels which are being used for sending. 409 // When the default channel (voe_channel) is used for sending, it is 410 // contained in send_channels_, otherwise not. 411 ChannelMap send_channels_; 412 uint32 default_receive_ssrc_; 413 // Note the default channel (voe_channel()) can reside in both 414 // receive_channels_ and send_channels_ in non-conference mode and in that 415 // case it will only be there if a non-zero default_receive_ssrc_ is set. 416 ChannelMap receive_channels_; // for multiple sources 417 // receive_channels_ can be read from WebRtc callback thread. Access from 418 // the WebRtc thread must be synchronized with edits on the worker thread. 419 // Reads on the worker thread are ok. 420 // 421 // Do not lock this on the VoE media processor thread; potential for deadlock 422 // exists. 423 mutable talk_base::CriticalSection receive_channels_cs_; 424 }; 425 426 } // namespace cricket 427 428 #endif // TALK_MEDIA_WEBRTCVOICEENGINE_H_ 429
