Home | History | Annotate | Download | only in webrtc 
      1 /*
      2  * libjingle
      3  * Copyright 2004 Google Inc.
      4  *
      5  * Redistribution and use in source and binary forms, with or without
      6  * modification, are permitted provided that the following conditions are met:
      7  *
      8  *  1. Redistributions of source code must retain the above copyright notice,
      9  *     this list of conditions and the following disclaimer.
     10  *  2. Redistributions in binary form must reproduce the above copyright notice,
     11  *     this list of conditions and the following disclaimer in the documentation
     12  *     and/or other materials provided with the distribution.
     13  *  3. The name of the author may not be used to endorse or promote products
     14  *     derived from this software without specific prior written permission.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
     17  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
     18  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
     19  * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     20  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
     21  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
     22  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
     23  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
     24  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
     25  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26  */
     27 
     28 #ifndef TALK_MEDIA_WEBRTCVOICEENGINE_H_
     29 #define TALK_MEDIA_WEBRTCVOICEENGINE_H_
     30 
     31 #include <map>
     32 #include <set>
     33 #include <string>
     34 #include <vector>
     35 
     36 #include "talk/media/base/rtputils.h"
     37 #include "talk/media/webrtc/webrtccommon.h"
     38 #include "talk/media/webrtc/webrtcexport.h"
     39 #include "talk/media/webrtc/webrtcvoe.h"
     40 #include "talk/session/media/channel.h"
     41 #include "webrtc/base/buffer.h"
     42 #include "webrtc/base/byteorder.h"
     43 #include "webrtc/base/logging.h"
     44 #include "webrtc/base/scoped_ptr.h"
     45 #include "webrtc/base/stream.h"
     46 #include "webrtc/common.h"
     47 
     48 #if !defined(LIBPEERCONNECTION_LIB) && \
     49     !defined(LIBPEERCONNECTION_IMPLEMENTATION)
     50 // If you hit this, then you've tried to include this header from outside
     51 // the shared library.  An instance of this class must only be created from
     52 // within the library that actually implements it.  Otherwise use the
     53 // WebRtcMediaEngine to construct an instance.
     54 #error "Bogus include."
     55 #endif
     56 
     57 namespace webrtc {
     58 class VideoEngine;
     59 }
     60 
     61 namespace cricket {
     62 
     63 // WebRtcSoundclipStream is an adapter object that allows a memory stream to be
     64 // passed into WebRtc, and support looping.
     65 class WebRtcSoundclipStream : public webrtc::InStream {
     66  public:
     67   WebRtcSoundclipStream(const char* buf, size_t len)
     68       : mem_(buf, len), loop_(true) {
     69   }
     70   void set_loop(bool loop) { loop_ = loop; }
     71   virtual int Read(void* buf, int len);
     72   virtual int Rewind();
     73 
     74  private:
     75   rtc::MemoryStream mem_;
     76   bool loop_;
     77 };
     78 
     79 // WebRtcMonitorStream is used to monitor a stream coming from WebRtc.
     80 // For now we just dump the data.
     81 class WebRtcMonitorStream : public webrtc::OutStream {
     82   virtual bool Write(const void *buf, int len) {
     83     return true;
     84   }
     85 };
     86 
     87 class AudioDeviceModule;
     88 class AudioRenderer;
     89 class VoETraceWrapper;
     90 class VoEWrapper;
     91 class VoiceProcessor;
     92 class WebRtcSoundclipMedia;
     93 class WebRtcVoiceMediaChannel;
     94 
     95 // WebRtcVoiceEngine is a class to be used with CompositeMediaEngine.
     96 // It uses the WebRtc VoiceEngine library for audio handling.
     97 class WebRtcVoiceEngine
     98     : public webrtc::VoiceEngineObserver,
     99       public webrtc::TraceCallback,
    100       public webrtc::VoEMediaProcess  {
    101  public:
    102   WebRtcVoiceEngine();
    103   // Dependency injection for testing.
    104   WebRtcVoiceEngine(VoEWrapper* voe_wrapper,
    105                     VoEWrapper* voe_wrapper_sc,
    106                     VoETraceWrapper* tracing);
    107   ~WebRtcVoiceEngine();
    108   bool Init(rtc::Thread* worker_thread);
    109   void Terminate();
    110 
    111   int GetCapabilities();
    112   VoiceMediaChannel* CreateChannel();
    113 
    114   SoundclipMedia* CreateSoundclip();
    115 
    116   AudioOptions GetOptions() const { return options_; }
    117   bool SetOptions(const AudioOptions& options);
    118   // Overrides, when set, take precedence over the options on a
    119   // per-option basis.  For example, if AGC is set in options and AEC
    120   // is set in overrides, AGC and AEC will be both be set.  Overrides
    121   // can also turn off options.  For example, if AGC is set to "on" in
    122   // options and AGC is set to "off" in overrides, the result is that
    123   // AGC will be off until different overrides are applied or until
    124   // the overrides are cleared.  Only one set of overrides is present
    125   // at a time (they do not "stack").  And when the overrides are
    126   // cleared, the media engine's state reverts back to the options set
    127   // via SetOptions.  This allows us to have both "persistent options"
    128   // (the normal options) and "temporary options" (overrides).
    129   bool SetOptionOverrides(const AudioOptions& options);
    130   bool ClearOptionOverrides();
    131   bool SetDelayOffset(int offset);
    132   bool SetDevices(const Device* in_device, const Device* out_device);
    133   bool GetOutputVolume(int* level);
    134   bool SetOutputVolume(int level);
    135   int GetInputLevel();
    136   bool SetLocalMonitor(bool enable);
    137 
    138   const std::vector<AudioCodec>& codecs();
    139   bool FindCodec(const AudioCodec& codec);
    140   bool FindWebRtcCodec(const AudioCodec& codec, webrtc::CodecInst* gcodec);
    141 
    142   const std::vector<RtpHeaderExtension>& rtp_header_extensions() const;
    143 
    144   void SetLogging(int min_sev, const char* filter);
    145 
    146   bool RegisterProcessor(uint32 ssrc,
    147                          VoiceProcessor* voice_processor,
    148                          MediaProcessorDirection direction);
    149   bool UnregisterProcessor(uint32 ssrc,
    150                            VoiceProcessor* voice_processor,
    151                            MediaProcessorDirection direction);
    152 
    153   // Method from webrtc::VoEMediaProcess
    154   virtual void Process(int channel,
    155                        webrtc::ProcessingTypes type,
    156                        int16_t audio10ms[],
    157                        int length,
    158                        int sampling_freq,
    159                        bool is_stereo);
    160 
    161   // For tracking WebRtc channels. Needed because we have to pause them
    162   // all when switching devices.
    163   // May only be called by WebRtcVoiceMediaChannel.
    164   void RegisterChannel(WebRtcVoiceMediaChannel *channel);
    165   void UnregisterChannel(WebRtcVoiceMediaChannel *channel);
    166 
    167   // May only be called by WebRtcSoundclipMedia.
    168   void RegisterSoundclip(WebRtcSoundclipMedia *channel);
    169   void UnregisterSoundclip(WebRtcSoundclipMedia *channel);
    170 
    171   // Called by WebRtcVoiceMediaChannel to set a gain offset from
    172   // the default AGC target level.
    173   bool AdjustAgcLevel(int delta);
    174 
    175   VoEWrapper* voe() { return voe_wrapper_.get(); }
    176   VoEWrapper* voe_sc() { return voe_wrapper_sc_.get(); }
    177   int GetLastEngineError();
    178 
    179   // Set the external ADMs. This can only be called before Init.
    180   bool SetAudioDeviceModule(webrtc::AudioDeviceModule* adm,
    181                             webrtc::AudioDeviceModule* adm_sc);
    182 
    183   // Starts AEC dump using existing file.
    184   bool StartAecDump(rtc::PlatformFile file);
    185 
    186   // Check whether the supplied trace should be ignored.
    187   bool ShouldIgnoreTrace(const std::string& trace);
    188 
    189   // Create a VoiceEngine Channel.
    190   int CreateMediaVoiceChannel();
    191   int CreateSoundclipVoiceChannel();
    192 
    193  private:
    194   typedef std::vector<WebRtcSoundclipMedia *> SoundclipList;
    195   typedef std::vector<WebRtcVoiceMediaChannel *> ChannelList;
    196   typedef sigslot::
    197       signal3<uint32, MediaProcessorDirection, AudioFrame*> FrameSignal;
    198 
    199   void Construct();
    200   void ConstructCodecs();
    201   bool InitInternal();
    202   bool EnsureSoundclipEngineInit();
    203   void SetTraceFilter(int filter);
    204   void SetTraceOptions(const std::string& options);
    205   // Applies either options or overrides.  Every option that is "set"
    206   // will be applied.  Every option not "set" will be ignored.  This
    207   // allows us to selectively turn on and off different options easily
    208   // at any time.
    209   bool ApplyOptions(const AudioOptions& options);
    210   virtual void Print(webrtc::TraceLevel level, const char* trace, int length);
    211   virtual void CallbackOnError(int channel, int errCode);
    212   // Given the device type, name, and id, find device id. Return true and
    213   // set the output parameter rtc_id if successful.
    214   bool FindWebRtcAudioDeviceId(
    215       bool is_input, const std::string& dev_name, int dev_id, int* rtc_id);
    216   bool FindChannelAndSsrc(int channel_num,
    217                           WebRtcVoiceMediaChannel** channel,
    218                           uint32* ssrc) const;
    219   bool FindChannelNumFromSsrc(uint32 ssrc,
    220                               MediaProcessorDirection direction,
    221                               int* channel_num);
    222   bool ChangeLocalMonitor(bool enable);
    223   bool PauseLocalMonitor();
    224   bool ResumeLocalMonitor();
    225 
    226   bool UnregisterProcessorChannel(MediaProcessorDirection channel_direction,
    227                                   uint32 ssrc,
    228                                   VoiceProcessor* voice_processor,
    229                                   MediaProcessorDirection processor_direction);
    230 
    231   void StartAecDump(const std::string& filename);
    232   void StopAecDump();
    233   int CreateVoiceChannel(VoEWrapper* voe);
    234 
    235   // When a voice processor registers with the engine, it is connected
    236   // to either the Rx or Tx signals, based on the direction parameter.
    237   // SignalXXMediaFrame will be invoked for every audio packet.
    238   FrameSignal SignalRxMediaFrame;
    239   FrameSignal SignalTxMediaFrame;
    240 
    241   static const int kDefaultLogSeverity = rtc::LS_WARNING;
    242 
    243   // The primary instance of WebRtc VoiceEngine.
    244   rtc::scoped_ptr<VoEWrapper> voe_wrapper_;
    245   // A secondary instance, for playing out soundclips (on the 'ring' device).
    246   rtc::scoped_ptr<VoEWrapper> voe_wrapper_sc_;
    247   bool voe_wrapper_sc_initialized_;
    248   rtc::scoped_ptr<VoETraceWrapper> tracing_;
    249   // The external audio device manager
    250   webrtc::AudioDeviceModule* adm_;
    251   webrtc::AudioDeviceModule* adm_sc_;
    252   int log_filter_;
    253   std::string log_options_;
    254   bool is_dumping_aec_;
    255   std::vector<AudioCodec> codecs_;
    256   std::vector<RtpHeaderExtension> rtp_header_extensions_;
    257   bool desired_local_monitor_enable_;
    258   rtc::scoped_ptr<WebRtcMonitorStream> monitor_;
    259   SoundclipList soundclips_;
    260   ChannelList channels_;
    261   // channels_ can be read from WebRtc callback thread. We need a lock on that
    262   // callback as well as the RegisterChannel/UnregisterChannel.
    263   rtc::CriticalSection channels_cs_;
    264   webrtc::AgcConfig default_agc_config_;
    265 
    266   webrtc::Config voe_config_;
    267 
    268   bool initialized_;
    269   // See SetOptions and SetOptionOverrides for a description of the
    270   // difference between options and overrides.
    271   // options_ are the base options, which combined with the
    272   // option_overrides_, create the current options being used.
    273   // options_ is stored so that when option_overrides_ is cleared, we
    274   // can restore the options_ without the option_overrides.
    275   AudioOptions options_;
    276   AudioOptions option_overrides_;
    277 
    278   // When the media processor registers with the engine, the ssrc is cached
    279   // here so that a look up need not be made when the callback is invoked.
    280   // This is necessary because the lookup results in mux_channels_cs lock being
    281   // held and if a remote participant leaves the hangout at the same time
    282   // we hit a deadlock.
    283   uint32 tx_processor_ssrc_;
    284   uint32 rx_processor_ssrc_;
    285 
    286   rtc::CriticalSection signal_media_critical_;
    287 
    288   // Cache received experimental_aec and experimental_ns values, and apply them
    289   // in case they are missing in the audio options. We need to do this because
    290   // SetExtraOptions() will revert to defaults for options which are not
    291   // provided.
    292   Settable<bool> experimental_aec_;
    293   Settable<bool> experimental_ns_;
    294 };
    295 
    296 // WebRtcMediaChannel is a class that implements the common WebRtc channel
    297 // functionality.
    298 template <class T, class E>
    299 class WebRtcMediaChannel : public T, public webrtc::Transport {
    300  public:
    301   WebRtcMediaChannel(E *engine, int channel)
    302       : engine_(engine), voe_channel_(channel) {}
    303   E *engine() { return engine_; }
    304   int voe_channel() const { return voe_channel_; }
    305   bool valid() const { return voe_channel_ != -1; }
    306 
    307  protected:
    308   // implements Transport interface
    309   virtual int SendPacket(int channel, const void *data, int len) {
    310     rtc::Buffer packet(data, len, kMaxRtpPacketLen);
    311     if (!T::SendPacket(&packet)) {
    312       return -1;
    313     }
    314     return len;
    315   }
    316 
    317   virtual int SendRTCPPacket(int channel, const void *data, int len) {
    318     rtc::Buffer packet(data, len, kMaxRtpPacketLen);
    319     return T::SendRtcp(&packet) ? len : -1;
    320   }
    321 
    322  private:
    323   E *engine_;
    324   int voe_channel_;
    325 };
    326 
    327 // WebRtcVoiceMediaChannel is an implementation of VoiceMediaChannel that uses
    328 // WebRtc Voice Engine.
    329 class WebRtcVoiceMediaChannel
    330     : public WebRtcMediaChannel<VoiceMediaChannel, WebRtcVoiceEngine> {
    331  public:
    332   explicit WebRtcVoiceMediaChannel(WebRtcVoiceEngine *engine);
    333   virtual ~WebRtcVoiceMediaChannel();
    334   virtual bool SetOptions(const AudioOptions& options);
    335   virtual bool GetOptions(AudioOptions* options) const {
    336     *options = options_;
    337     return true;
    338   }
    339   virtual bool SetRecvCodecs(const std::vector<AudioCodec> &codecs);
    340   virtual bool SetSendCodecs(const std::vector<AudioCodec> &codecs);
    341   virtual bool SetRecvRtpHeaderExtensions(
    342       const std::vector<RtpHeaderExtension>& extensions);
    343   virtual bool SetSendRtpHeaderExtensions(
    344       const std::vector<RtpHeaderExtension>& extensions);
    345   virtual bool SetPlayout(bool playout);
    346   bool PausePlayout();
    347   bool ResumePlayout();
    348   virtual bool SetSend(SendFlags send);
    349   bool PauseSend();
    350   bool ResumeSend();
    351   virtual bool AddSendStream(const StreamParams& sp);
    352   virtual bool RemoveSendStream(uint32 ssrc);
    353   virtual bool AddRecvStream(const StreamParams& sp);
    354   virtual bool RemoveRecvStream(uint32 ssrc);
    355   virtual bool SetRemoteRenderer(uint32 ssrc, AudioRenderer* renderer);
    356   virtual bool SetLocalRenderer(uint32 ssrc, AudioRenderer* renderer);
    357   virtual bool GetActiveStreams(AudioInfo::StreamList* actives);
    358   virtual int GetOutputLevel();
    359   virtual int GetTimeSinceLastTyping();
    360   virtual void SetTypingDetectionParameters(int time_window,
    361       int cost_per_typing, int reporting_threshold, int penalty_decay,
    362       int type_event_delay);
    363   virtual bool SetOutputScaling(uint32 ssrc, double left, double right);
    364   virtual bool GetOutputScaling(uint32 ssrc, double* left, double* right);
    365 
    366   virtual bool SetRingbackTone(const char *buf, int len);
    367   virtual bool PlayRingbackTone(uint32 ssrc, bool play, bool loop);
    368   virtual bool CanInsertDtmf();
    369   virtual bool InsertDtmf(uint32 ssrc, int event, int duration, int flags);
    370 
    371   virtual void OnPacketReceived(rtc::Buffer* packet,
    372                                 const rtc::PacketTime& packet_time);
    373   virtual void OnRtcpReceived(rtc::Buffer* packet,
    374                               const rtc::PacketTime& packet_time);
    375   virtual void OnReadyToSend(bool ready) {}
    376   virtual bool MuteStream(uint32 ssrc, bool on);
    377   virtual bool SetStartSendBandwidth(int bps);
    378   virtual bool SetMaxSendBandwidth(int bps);
    379   virtual bool GetStats(VoiceMediaInfo* info);
    380   // Gets last reported error from WebRtc voice engine.  This should be only
    381   // called in response a failure.
    382   virtual void GetLastMediaError(uint32* ssrc,
    383                                  VoiceMediaChannel::Error* error);
    384   bool FindSsrc(int channel_num, uint32* ssrc);
    385   void OnError(uint32 ssrc, int error);
    386 
    387   bool sending() const { return send_ != SEND_NOTHING; }
    388   int GetReceiveChannelNum(uint32 ssrc);
    389   int GetSendChannelNum(uint32 ssrc);
    390 
    391   bool SetupSharedBandwidthEstimation(webrtc::VideoEngine* vie,
    392                                       int vie_channel);
    393  protected:
    394   int GetLastEngineError() { return engine()->GetLastEngineError(); }
    395   int GetOutputLevel(int channel);
    396   bool GetRedSendCodec(const AudioCodec& red_codec,
    397                        const std::vector<AudioCodec>& all_codecs,
    398                        webrtc::CodecInst* send_codec);
    399   bool EnableRtcp(int channel);
    400   bool ResetRecvCodecs(int channel);
    401   bool SetPlayout(int channel, bool playout);
    402   static uint32 ParseSsrc(const void* data, size_t len, bool rtcp);
    403   static Error WebRtcErrorToChannelError(int err_code);
    404 
    405  private:
    406   class WebRtcVoiceChannelRenderer;
    407   // Map of ssrc to WebRtcVoiceChannelRenderer object.  A new object of
    408   // WebRtcVoiceChannelRenderer will be created for every new stream and
    409   // will be destroyed when the stream goes away.
    410   typedef std::map<uint32, WebRtcVoiceChannelRenderer*> ChannelMap;
    411   typedef int (webrtc::VoERTP_RTCP::* ExtensionSetterFunction)(int, bool,
    412       unsigned char);
    413 
    414   void SetNack(int channel, bool nack_enabled);
    415   void SetNack(const ChannelMap& channels, bool nack_enabled);
    416   bool SetSendCodec(const webrtc::CodecInst& send_codec);
    417   bool SetSendCodec(int channel, const webrtc::CodecInst& send_codec);
    418   bool ChangePlayout(bool playout);
    419   bool ChangeSend(SendFlags send);
    420   bool ChangeSend(int channel, SendFlags send);
    421   void ConfigureSendChannel(int channel);
    422   bool ConfigureRecvChannel(int channel);
    423   bool DeleteChannel(int channel);
    424   bool InConferenceMode() const {
    425     return options_.conference_mode.GetWithDefaultIfUnset(false);
    426   }
    427   bool IsDefaultChannel(int channel_id) const {
    428     return channel_id == voe_channel();
    429   }
    430   bool SetSendCodecs(int channel, const std::vector<AudioCodec>& codecs);
    431   bool SetSendBandwidthInternal(int bps);
    432 
    433   bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id,
    434                           const RtpHeaderExtension* extension);
    435   bool SetupSharedBweOnChannel(int voe_channel);
    436 
    437   bool SetChannelRecvRtpHeaderExtensions(
    438     int channel_id,
    439     const std::vector<RtpHeaderExtension>& extensions);
    440   bool SetChannelSendRtpHeaderExtensions(
    441     int channel_id,
    442     const std::vector<RtpHeaderExtension>& extensions);
    443 
    444   rtc::scoped_ptr<WebRtcSoundclipStream> ringback_tone_;
    445   std::set<int> ringback_channels_;  // channels playing ringback
    446   std::vector<AudioCodec> recv_codecs_;
    447   std::vector<AudioCodec> send_codecs_;
    448   rtc::scoped_ptr<webrtc::CodecInst> send_codec_;
    449   bool send_bw_setting_;
    450   int send_bw_bps_;
    451   AudioOptions options_;
    452   bool dtmf_allowed_;
    453   bool desired_playout_;
    454   bool nack_enabled_;
    455   bool playout_;
    456   bool typing_noise_detected_;
    457   SendFlags desired_send_;
    458   SendFlags send_;
    459   // shared_bwe_vie_ and shared_bwe_vie_channel_ together identifies a WebRTC
    460   // VideoEngine channel that this voice channel should forward incoming packets
    461   // to for Bandwidth Estimation purposes.
    462   webrtc::VideoEngine* shared_bwe_vie_;
    463   int shared_bwe_vie_channel_;
    464 
    465   // send_channels_ contains the channels which are being used for sending.
    466   // When the default channel (voe_channel) is used for sending, it is
    467   // contained in send_channels_, otherwise not.
    468   ChannelMap send_channels_;
    469   std::vector<RtpHeaderExtension> send_extensions_;
    470   uint32 default_receive_ssrc_;
    471   // Note the default channel (voe_channel()) can reside in both
    472   // receive_channels_ and send_channels_ in non-conference mode and in that
    473   // case it will only be there if a non-zero default_receive_ssrc_ is set.
    474   ChannelMap receive_channels_;  // for multiple sources
    475   // receive_channels_ can be read from WebRtc callback thread.  Access from
    476   // the WebRtc thread must be synchronized with edits on the worker thread.
    477   // Reads on the worker thread are ok.
    478   //
    479   std::vector<RtpHeaderExtension> receive_extensions_;
    480   // Do not lock this on the VoE media processor thread; potential for deadlock
    481   // exists.
    482   mutable rtc::CriticalSection receive_channels_cs_;
    483 };
    484 
    485 }  // namespace cricket
    486 
    487 #endif  // TALK_MEDIA_WEBRTCVOICEENGINE_H_
    488