Home | History | Annotate | Download | only in webrtc 
      1 /*
      2  * libjingle
      3  * Copyright 2004 Google Inc.
      4  *
      5  * Redistribution and use in source and binary forms, with or without
      6  * modification, are permitted provided that the following conditions are met:
      7  *
      8  *  1. Redistributions of source code must retain the above copyright notice,
      9  *     this list of conditions and the following disclaimer.
     10  *  2. Redistributions in binary form must reproduce the above copyright notice,
     11  *     this list of conditions and the following disclaimer in the documentation
     12  *     and/or other materials provided with the distribution.
     13  *  3. The name of the author may not be used to endorse or promote products
     14  *     derived from this software without specific prior written permission.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
     17  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
     18  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
     19  * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     20  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
     21  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
     22  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
     23  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
     24  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
     25  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26  */
     27 
     28 #ifndef TALK_MEDIA_WEBRTCVOICEENGINE_H_
     29 #define TALK_MEDIA_WEBRTCVOICEENGINE_H_
     30 
     31 #include <map>
     32 #include <set>
     33 #include <string>
     34 #include <vector>
     35 
     36 #include "talk/base/buffer.h"
     37 #include "talk/base/byteorder.h"
     38 #include "talk/base/logging.h"
     39 #include "talk/base/scoped_ptr.h"
     40 #include "talk/base/stream.h"
     41 #include "talk/media/base/rtputils.h"
     42 #include "talk/media/webrtc/webrtccommon.h"
     43 #include "talk/media/webrtc/webrtcexport.h"
     44 #include "talk/media/webrtc/webrtcvoe.h"
     45 #include "talk/session/media/channel.h"
     46 #include "webrtc/common.h"
     47 
     48 #if !defined(LIBPEERCONNECTION_LIB) && \
     49     !defined(LIBPEERCONNECTION_IMPLEMENTATION)
     50 #error "Bogus include."
     51 #endif
     52 
     53 namespace cricket {
     54 
     55 // WebRtcSoundclipStream is an adapter object that allows a memory stream to be
     56 // passed into WebRtc, and support looping.
     57 class WebRtcSoundclipStream : public webrtc::InStream {
     58  public:
     59   WebRtcSoundclipStream(const char* buf, size_t len)
     60       : mem_(buf, len), loop_(true) {
     61   }
     62   void set_loop(bool loop) { loop_ = loop; }
     63   virtual int Read(void* buf, int len);
     64   virtual int Rewind();
     65 
     66  private:
     67   talk_base::MemoryStream mem_;
     68   bool loop_;
     69 };
     70 
     71 // WebRtcMonitorStream is used to monitor a stream coming from WebRtc.
     72 // For now we just dump the data.
     73 class WebRtcMonitorStream : public webrtc::OutStream {
     74   virtual bool Write(const void *buf, int len) {
     75     return true;
     76   }
     77 };
     78 
     79 class AudioDeviceModule;
     80 class AudioRenderer;
     81 class VoETraceWrapper;
     82 class VoEWrapper;
     83 class VoiceProcessor;
     84 class WebRtcSoundclipMedia;
     85 class WebRtcVoiceMediaChannel;
     86 
     87 // WebRtcVoiceEngine is a class to be used with CompositeMediaEngine.
     88 // It uses the WebRtc VoiceEngine library for audio handling.
     89 class WebRtcVoiceEngine
     90     : public webrtc::VoiceEngineObserver,
     91       public webrtc::TraceCallback,
     92       public webrtc::VoEMediaProcess  {
     93  public:
     94   WebRtcVoiceEngine();
     95   // Dependency injection for testing.
     96   WebRtcVoiceEngine(VoEWrapper* voe_wrapper,
     97                     VoEWrapper* voe_wrapper_sc,
     98                     VoETraceWrapper* tracing);
     99   ~WebRtcVoiceEngine();
    100   bool Init(talk_base::Thread* worker_thread);
    101   void Terminate();
    102 
    103   int GetCapabilities();
    104   VoiceMediaChannel* CreateChannel();
    105 
    106   SoundclipMedia* CreateSoundclip();
    107 
    108   AudioOptions GetOptions() const { return options_; }
    109   bool SetOptions(const AudioOptions& options);
    110   // Overrides, when set, take precedence over the options on a
    111   // per-option basis.  For example, if AGC is set in options and AEC
    112   // is set in overrides, AGC and AEC will be both be set.  Overrides
    113   // can also turn off options.  For example, if AGC is set to "on" in
    114   // options and AGC is set to "off" in overrides, the result is that
    115   // AGC will be off until different overrides are applied or until
    116   // the overrides are cleared.  Only one set of overrides is present
    117   // at a time (they do not "stack").  And when the overrides are
    118   // cleared, the media engine's state reverts back to the options set
    119   // via SetOptions.  This allows us to have both "persistent options"
    120   // (the normal options) and "temporary options" (overrides).
    121   bool SetOptionOverrides(const AudioOptions& options);
    122   bool ClearOptionOverrides();
    123   bool SetDelayOffset(int offset);
    124   bool SetDevices(const Device* in_device, const Device* out_device);
    125   bool GetOutputVolume(int* level);
    126   bool SetOutputVolume(int level);
    127   int GetInputLevel();
    128   bool SetLocalMonitor(bool enable);
    129 
    130   const std::vector<AudioCodec>& codecs();
    131   bool FindCodec(const AudioCodec& codec);
    132   bool FindWebRtcCodec(const AudioCodec& codec, webrtc::CodecInst* gcodec);
    133 
    134   const std::vector<RtpHeaderExtension>& rtp_header_extensions() const;
    135 
    136   void SetLogging(int min_sev, const char* filter);
    137 
    138   bool RegisterProcessor(uint32 ssrc,
    139                          VoiceProcessor* voice_processor,
    140                          MediaProcessorDirection direction);
    141   bool UnregisterProcessor(uint32 ssrc,
    142                            VoiceProcessor* voice_processor,
    143                            MediaProcessorDirection direction);
    144 
    145   // Method from webrtc::VoEMediaProcess
    146   virtual void Process(int channel,
    147                        webrtc::ProcessingTypes type,
    148                        int16_t audio10ms[],
    149                        int length,
    150                        int sampling_freq,
    151                        bool is_stereo);
    152 
    153   // For tracking WebRtc channels. Needed because we have to pause them
    154   // all when switching devices.
    155   // May only be called by WebRtcVoiceMediaChannel.
    156   void RegisterChannel(WebRtcVoiceMediaChannel *channel);
    157   void UnregisterChannel(WebRtcVoiceMediaChannel *channel);
    158 
    159   // May only be called by WebRtcSoundclipMedia.
    160   void RegisterSoundclip(WebRtcSoundclipMedia *channel);
    161   void UnregisterSoundclip(WebRtcSoundclipMedia *channel);
    162 
    163   // Called by WebRtcVoiceMediaChannel to set a gain offset from
    164   // the default AGC target level.
    165   bool AdjustAgcLevel(int delta);
    166 
    167   VoEWrapper* voe() { return voe_wrapper_.get(); }
    168   VoEWrapper* voe_sc() { return voe_wrapper_sc_.get(); }
    169   int GetLastEngineError();
    170 
    171   // Set the external ADMs. This can only be called before Init.
    172   bool SetAudioDeviceModule(webrtc::AudioDeviceModule* adm,
    173                             webrtc::AudioDeviceModule* adm_sc);
    174 
    175   // Starts AEC dump using existing file.
    176   bool StartAecDump(talk_base::PlatformFile file);
    177 
    178   // Check whether the supplied trace should be ignored.
    179   bool ShouldIgnoreTrace(const std::string& trace);
    180 
    181   // Create a VoiceEngine Channel.
    182   int CreateMediaVoiceChannel();
    183   int CreateSoundclipVoiceChannel();
    184 
    185  private:
    186   typedef std::vector<WebRtcSoundclipMedia *> SoundclipList;
    187   typedef std::vector<WebRtcVoiceMediaChannel *> ChannelList;
    188   typedef sigslot::
    189       signal3<uint32, MediaProcessorDirection, AudioFrame*> FrameSignal;
    190 
    191   void Construct();
    192   void ConstructCodecs();
    193   bool InitInternal();
    194   bool EnsureSoundclipEngineInit();
    195   void SetTraceFilter(int filter);
    196   void SetTraceOptions(const std::string& options);
    197   // Applies either options or overrides.  Every option that is "set"
    198   // will be applied.  Every option not "set" will be ignored.  This
    199   // allows us to selectively turn on and off different options easily
    200   // at any time.
    201   bool ApplyOptions(const AudioOptions& options);
    202   virtual void Print(webrtc::TraceLevel level, const char* trace, int length);
    203   virtual void CallbackOnError(int channel, int errCode);
    204   // Given the device type, name, and id, find device id. Return true and
    205   // set the output parameter rtc_id if successful.
    206   bool FindWebRtcAudioDeviceId(
    207       bool is_input, const std::string& dev_name, int dev_id, int* rtc_id);
    208   bool FindChannelAndSsrc(int channel_num,
    209                           WebRtcVoiceMediaChannel** channel,
    210                           uint32* ssrc) const;
    211   bool FindChannelNumFromSsrc(uint32 ssrc,
    212                               MediaProcessorDirection direction,
    213                               int* channel_num);
    214   bool ChangeLocalMonitor(bool enable);
    215   bool PauseLocalMonitor();
    216   bool ResumeLocalMonitor();
    217 
    218   bool UnregisterProcessorChannel(MediaProcessorDirection channel_direction,
    219                                   uint32 ssrc,
    220                                   VoiceProcessor* voice_processor,
    221                                   MediaProcessorDirection processor_direction);
    222 
    223   void StartAecDump(const std::string& filename);
    224   void StopAecDump();
    225   int CreateVoiceChannel(VoEWrapper* voe);
    226 
    227   // When a voice processor registers with the engine, it is connected
    228   // to either the Rx or Tx signals, based on the direction parameter.
    229   // SignalXXMediaFrame will be invoked for every audio packet.
    230   FrameSignal SignalRxMediaFrame;
    231   FrameSignal SignalTxMediaFrame;
    232 
    233   static const int kDefaultLogSeverity = talk_base::LS_WARNING;
    234 
    235   // The primary instance of WebRtc VoiceEngine.
    236   talk_base::scoped_ptr<VoEWrapper> voe_wrapper_;
    237   // A secondary instance, for playing out soundclips (on the 'ring' device).
    238   talk_base::scoped_ptr<VoEWrapper> voe_wrapper_sc_;
    239   bool voe_wrapper_sc_initialized_;
    240   talk_base::scoped_ptr<VoETraceWrapper> tracing_;
    241   // The external audio device manager
    242   webrtc::AudioDeviceModule* adm_;
    243   webrtc::AudioDeviceModule* adm_sc_;
    244   int log_filter_;
    245   std::string log_options_;
    246   bool is_dumping_aec_;
    247   std::vector<AudioCodec> codecs_;
    248   std::vector<RtpHeaderExtension> rtp_header_extensions_;
    249   bool desired_local_monitor_enable_;
    250   talk_base::scoped_ptr<WebRtcMonitorStream> monitor_;
    251   SoundclipList soundclips_;
    252   ChannelList channels_;
    253   // channels_ can be read from WebRtc callback thread. We need a lock on that
    254   // callback as well as the RegisterChannel/UnregisterChannel.
    255   talk_base::CriticalSection channels_cs_;
    256   webrtc::AgcConfig default_agc_config_;
    257 
    258   webrtc::Config voe_config_;
    259 
    260   bool initialized_;
    261   // See SetOptions and SetOptionOverrides for a description of the
    262   // difference between options and overrides.
    263   // options_ are the base options, which combined with the
    264   // option_overrides_, create the current options being used.
    265   // options_ is stored so that when option_overrides_ is cleared, we
    266   // can restore the options_ without the option_overrides.
    267   AudioOptions options_;
    268   AudioOptions option_overrides_;
    269 
    270   // When the media processor registers with the engine, the ssrc is cached
    271   // here so that a look up need not be made when the callback is invoked.
    272   // This is necessary because the lookup results in mux_channels_cs lock being
    273   // held and if a remote participant leaves the hangout at the same time
    274   // we hit a deadlock.
    275   uint32 tx_processor_ssrc_;
    276   uint32 rx_processor_ssrc_;
    277 
    278   talk_base::CriticalSection signal_media_critical_;
    279 };
    280 
    281 // WebRtcMediaChannel is a class that implements the common WebRtc channel
    282 // functionality.
    283 template <class T, class E>
    284 class WebRtcMediaChannel : public T, public webrtc::Transport {
    285  public:
    286   WebRtcMediaChannel(E *engine, int channel)
    287       : engine_(engine), voe_channel_(channel) {}
    288   E *engine() { return engine_; }
    289   int voe_channel() const { return voe_channel_; }
    290   bool valid() const { return voe_channel_ != -1; }
    291 
    292  protected:
    293   // implements Transport interface
    294   virtual int SendPacket(int channel, const void *data, int len) {
    295     talk_base::Buffer packet(data, len, kMaxRtpPacketLen);
    296     if (!T::SendPacket(&packet)) {
    297       return -1;
    298     }
    299     return len;
    300   }
    301 
    302   virtual int SendRTCPPacket(int channel, const void *data, int len) {
    303     talk_base::Buffer packet(data, len, kMaxRtpPacketLen);
    304     return T::SendRtcp(&packet) ? len : -1;
    305   }
    306 
    307  private:
    308   E *engine_;
    309   int voe_channel_;
    310 };
    311 
    312 // WebRtcVoiceMediaChannel is an implementation of VoiceMediaChannel that uses
    313 // WebRtc Voice Engine.
    314 class WebRtcVoiceMediaChannel
    315     : public WebRtcMediaChannel<VoiceMediaChannel, WebRtcVoiceEngine> {
    316  public:
    317   explicit WebRtcVoiceMediaChannel(WebRtcVoiceEngine *engine);
    318   virtual ~WebRtcVoiceMediaChannel();
    319   virtual bool SetOptions(const AudioOptions& options);
    320   virtual bool GetOptions(AudioOptions* options) const {
    321     *options = options_;
    322     return true;
    323   }
    324   virtual bool SetRecvCodecs(const std::vector<AudioCodec> &codecs);
    325   virtual bool SetSendCodecs(const std::vector<AudioCodec> &codecs);
    326   virtual bool SetRecvRtpHeaderExtensions(
    327       const std::vector<RtpHeaderExtension>& extensions);
    328   virtual bool SetSendRtpHeaderExtensions(
    329       const std::vector<RtpHeaderExtension>& extensions);
    330   virtual bool SetPlayout(bool playout);
    331   bool PausePlayout();
    332   bool ResumePlayout();
    333   virtual bool SetSend(SendFlags send);
    334   bool PauseSend();
    335   bool ResumeSend();
    336   virtual bool AddSendStream(const StreamParams& sp);
    337   virtual bool RemoveSendStream(uint32 ssrc);
    338   virtual bool AddRecvStream(const StreamParams& sp);
    339   virtual bool RemoveRecvStream(uint32 ssrc);
    340   virtual bool SetRemoteRenderer(uint32 ssrc, AudioRenderer* renderer);
    341   virtual bool SetLocalRenderer(uint32 ssrc, AudioRenderer* renderer);
    342   virtual bool GetActiveStreams(AudioInfo::StreamList* actives);
    343   virtual int GetOutputLevel();
    344   virtual int GetTimeSinceLastTyping();
    345   virtual void SetTypingDetectionParameters(int time_window,
    346       int cost_per_typing, int reporting_threshold, int penalty_decay,
    347       int type_event_delay);
    348   virtual bool SetOutputScaling(uint32 ssrc, double left, double right);
    349   virtual bool GetOutputScaling(uint32 ssrc, double* left, double* right);
    350 
    351   virtual bool SetRingbackTone(const char *buf, int len);
    352   virtual bool PlayRingbackTone(uint32 ssrc, bool play, bool loop);
    353   virtual bool CanInsertDtmf();
    354   virtual bool InsertDtmf(uint32 ssrc, int event, int duration, int flags);
    355 
    356   virtual void OnPacketReceived(talk_base::Buffer* packet,
    357                                 const talk_base::PacketTime& packet_time);
    358   virtual void OnRtcpReceived(talk_base::Buffer* packet,
    359                               const talk_base::PacketTime& packet_time);
    360   virtual void OnReadyToSend(bool ready) {}
    361   virtual bool MuteStream(uint32 ssrc, bool on);
    362   virtual bool SetStartSendBandwidth(int bps);
    363   virtual bool SetMaxSendBandwidth(int bps);
    364   virtual bool GetStats(VoiceMediaInfo* info);
    365   // Gets last reported error from WebRtc voice engine.  This should be only
    366   // called in response a failure.
    367   virtual void GetLastMediaError(uint32* ssrc,
    368                                  VoiceMediaChannel::Error* error);
    369   bool FindSsrc(int channel_num, uint32* ssrc);
    370   void OnError(uint32 ssrc, int error);
    371 
    372   bool sending() const { return send_ != SEND_NOTHING; }
    373   int GetReceiveChannelNum(uint32 ssrc);
    374   int GetSendChannelNum(uint32 ssrc);
    375 
    376  protected:
    377   int GetLastEngineError() { return engine()->GetLastEngineError(); }
    378   int GetOutputLevel(int channel);
    379   bool GetRedSendCodec(const AudioCodec& red_codec,
    380                        const std::vector<AudioCodec>& all_codecs,
    381                        webrtc::CodecInst* send_codec);
    382   bool EnableRtcp(int channel);
    383   bool ResetRecvCodecs(int channel);
    384   bool SetPlayout(int channel, bool playout);
    385   static uint32 ParseSsrc(const void* data, size_t len, bool rtcp);
    386   static Error WebRtcErrorToChannelError(int err_code);
    387 
    388  private:
    389   class WebRtcVoiceChannelRenderer;
    390   // Map of ssrc to WebRtcVoiceChannelRenderer object.  A new object of
    391   // WebRtcVoiceChannelRenderer will be created for every new stream and
    392   // will be destroyed when the stream goes away.
    393   typedef std::map<uint32, WebRtcVoiceChannelRenderer*> ChannelMap;
    394   typedef int (webrtc::VoERTP_RTCP::* ExtensionSetterFunction)(int, bool,
    395       unsigned char);
    396 
    397   void SetNack(int channel, bool nack_enabled);
    398   void SetNack(const ChannelMap& channels, bool nack_enabled);
    399   bool SetSendCodec(const webrtc::CodecInst& send_codec);
    400   bool SetSendCodec(int channel, const webrtc::CodecInst& send_codec);
    401   bool ChangePlayout(bool playout);
    402   bool ChangeSend(SendFlags send);
    403   bool ChangeSend(int channel, SendFlags send);
    404   void ConfigureSendChannel(int channel);
    405   bool ConfigureRecvChannel(int channel);
    406   bool DeleteChannel(int channel);
    407   bool InConferenceMode() const {
    408     return options_.conference_mode.GetWithDefaultIfUnset(false);
    409   }
    410   bool IsDefaultChannel(int channel_id) const {
    411     return channel_id == voe_channel();
    412   }
    413   bool SetSendCodecs(int channel, const std::vector<AudioCodec>& codecs);
    414   bool SetSendBandwidthInternal(int bps);
    415 
    416   bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id,
    417                           const RtpHeaderExtension* extension);
    418 
    419   bool SetChannelRecvRtpHeaderExtensions(
    420     int channel_id,
    421     const std::vector<RtpHeaderExtension>& extensions);
    422   bool SetChannelSendRtpHeaderExtensions(
    423     int channel_id,
    424     const std::vector<RtpHeaderExtension>& extensions);
    425 
    426   talk_base::scoped_ptr<WebRtcSoundclipStream> ringback_tone_;
    427   std::set<int> ringback_channels_;  // channels playing ringback
    428   std::vector<AudioCodec> recv_codecs_;
    429   std::vector<AudioCodec> send_codecs_;
    430   talk_base::scoped_ptr<webrtc::CodecInst> send_codec_;
    431   bool send_bw_setting_;
    432   int send_bw_bps_;
    433   AudioOptions options_;
    434   bool dtmf_allowed_;
    435   bool desired_playout_;
    436   bool nack_enabled_;
    437   bool playout_;
    438   bool typing_noise_detected_;
    439   SendFlags desired_send_;
    440   SendFlags send_;
    441 
    442   // send_channels_ contains the channels which are being used for sending.
    443   // When the default channel (voe_channel) is used for sending, it is
    444   // contained in send_channels_, otherwise not.
    445   ChannelMap send_channels_;
    446   std::vector<RtpHeaderExtension> send_extensions_;
    447   uint32 default_receive_ssrc_;
    448   // Note the default channel (voe_channel()) can reside in both
    449   // receive_channels_ and send_channels_ in non-conference mode and in that
    450   // case it will only be there if a non-zero default_receive_ssrc_ is set.
    451   ChannelMap receive_channels_;  // for multiple sources
    452   // receive_channels_ can be read from WebRtc callback thread.  Access from
    453   // the WebRtc thread must be synchronized with edits on the worker thread.
    454   // Reads on the worker thread are ok.
    455   //
    456   std::vector<RtpHeaderExtension> receive_extensions_;
    457   // Do not lock this on the VoE media processor thread; potential for deadlock
    458   // exists.
    459   mutable talk_base::CriticalSection receive_channels_cs_;
    460 };
    461 
    462 }  // namespace cricket
    463 
    464 #endif  // TALK_MEDIA_WEBRTCVOICEENGINE_H_
    465