Home | History | Annotate | Download | only in webrtc 
      1 /*
      2  * libjingle
      3  * Copyright 2004 Google Inc.
      4  *
      5  * Redistribution and use in source and binary forms, with or without
      6  * modification, are permitted provided that the following conditions are met:
      7  *
      8  *  1. Redistributions of source code must retain the above copyright notice,
      9  *     this list of conditions and the following disclaimer.
     10  *  2. Redistributions in binary form must reproduce the above copyright notice,
     11  *     this list of conditions and the following disclaimer in the documentation
     12  *     and/or other materials provided with the distribution.
     13  *  3. The name of the author may not be used to endorse or promote products
     14  *     derived from this software without specific prior written permission.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
     17  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
     18  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
     19  * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     20  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
     21  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
     22  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
     23  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
     24  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
     25  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26  */
     27 
     28 #ifndef TALK_MEDIA_WEBRTCVIDEOENGINE_H_
     29 #define TALK_MEDIA_WEBRTCVIDEOENGINE_H_
     30 
     31 #include <map>
     32 #include <vector>
     33 
     34 #include "talk/base/scoped_ptr.h"
     35 #include "talk/media/base/codec.h"
     36 #include "talk/media/base/videocommon.h"
     37 #include "talk/media/webrtc/webrtccommon.h"
     38 #include "talk/media/webrtc/webrtcexport.h"
     39 #include "talk/media/webrtc/webrtcvideoencoderfactory.h"
     40 #include "talk/session/media/channel.h"
     41 #include "webrtc/video_engine/include/vie_base.h"
     42 
     43 #if !defined(LIBPEERCONNECTION_LIB) && \
     44     !defined(LIBPEERCONNECTION_IMPLEMENTATION)
     45 #error "Bogus include."
     46 #endif
     47 
     48 namespace webrtc {
     49 class VideoCaptureModule;
     50 class VideoDecoder;
     51 class VideoEncoder;
     52 class VideoRender;
     53 class ViEExternalCapture;
     54 class ViERTP_RTCP;
     55 }
     56 
     57 namespace talk_base {
     58 class CpuMonitor;
     59 }  // namespace talk_base
     60 
     61 namespace cricket {
     62 
     63 class VideoCapturer;
     64 class VideoFrame;
     65 class VideoProcessor;
     66 class VideoRenderer;
     67 class ViETraceWrapper;
     68 class ViEWrapper;
     69 class VoiceMediaChannel;
     70 class WebRtcDecoderObserver;
     71 class WebRtcEncoderObserver;
     72 class WebRtcLocalStreamInfo;
     73 class WebRtcRenderAdapter;
     74 class WebRtcVideoChannelRecvInfo;
     75 class WebRtcVideoChannelSendInfo;
     76 class WebRtcVideoDecoderFactory;
     77 class WebRtcVideoEncoderFactory;
     78 class WebRtcVideoMediaChannel;
     79 class WebRtcVoiceEngine;
     80 
     81 struct CapturedFrame;
     82 struct Device;
     83 
     84 class WebRtcVideoEngine : public sigslot::has_slots<>,
     85                           public webrtc::TraceCallback,
     86                           public WebRtcVideoEncoderFactory::Observer {
     87  public:
     88   // Creates the WebRtcVideoEngine with internal VideoCaptureModule.
     89   WebRtcVideoEngine();
     90   // For testing purposes. Allows the WebRtcVoiceEngine,
     91   // ViEWrapper and CpuMonitor to be mocks.
     92   // TODO(juberti): Remove the 3-arg ctor once fake tracing is implemented.
     93   WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine,
     94                     ViEWrapper* vie_wrapper,
     95                     talk_base::CpuMonitor* cpu_monitor);
     96   WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine,
     97                     ViEWrapper* vie_wrapper,
     98                     ViETraceWrapper* tracing,
     99                     talk_base::CpuMonitor* cpu_monitor);
    100   ~WebRtcVideoEngine();
    101 
    102   // Basic video engine implementation.
    103   bool Init(talk_base::Thread* worker_thread);
    104   void Terminate();
    105 
    106   int GetCapabilities();
    107   bool SetOptions(int options);
    108   bool SetDefaultEncoderConfig(const VideoEncoderConfig& config);
    109 
    110   WebRtcVideoMediaChannel* CreateChannel(VoiceMediaChannel* voice_channel);
    111 
    112   const std::vector<VideoCodec>& codecs() const;
    113   const std::vector<RtpHeaderExtension>& rtp_header_extensions() const;
    114   void SetLogging(int min_sev, const char* filter);
    115 
    116   bool SetLocalRenderer(VideoRenderer* renderer);
    117   sigslot::repeater2<VideoCapturer*, CaptureState> SignalCaptureStateChange;
    118 
    119   // Set the VoiceEngine for A/V sync. This can only be called before Init.
    120   bool SetVoiceEngine(WebRtcVoiceEngine* voice_engine);
    121   // Set a WebRtcVideoDecoderFactory for external decoding. Video engine does
    122   // not take the ownership of |decoder_factory|. The caller needs to make sure
    123   // that |decoder_factory| outlives the video engine.
    124   void SetExternalDecoderFactory(WebRtcVideoDecoderFactory* decoder_factory);
    125   // Set a WebRtcVideoEncoderFactory for external encoding. Video engine does
    126   // not take the ownership of |encoder_factory|. The caller needs to make sure
    127   // that |encoder_factory| outlives the video engine.
    128   void SetExternalEncoderFactory(WebRtcVideoEncoderFactory* encoder_factory);
    129   // Enable the render module with timing control.
    130   bool EnableTimedRender();
    131 
    132   // Returns an external decoder for the given codec type. The return value
    133   // can be NULL if decoder factory is not given or it does not support the
    134   // codec type. The caller takes the ownership of the returned object.
    135   webrtc::VideoDecoder* CreateExternalDecoder(webrtc::VideoCodecType type);
    136   // Releases the decoder instance created by CreateExternalDecoder().
    137   void DestroyExternalDecoder(webrtc::VideoDecoder* decoder);
    138 
    139   // Returns an external encoder for the given codec type. The return value
    140   // can be NULL if encoder factory is not given or it does not support the
    141   // codec type. The caller takes the ownership of the returned object.
    142   webrtc::VideoEncoder* CreateExternalEncoder(webrtc::VideoCodecType type);
    143   // Releases the encoder instance created by CreateExternalEncoder().
    144   void DestroyExternalEncoder(webrtc::VideoEncoder* encoder);
    145 
    146   // Returns true if the codec type is supported by the external encoder.
    147   bool IsExternalEncoderCodecType(webrtc::VideoCodecType type) const;
    148 
    149   // Functions called by WebRtcVideoMediaChannel.
    150   talk_base::Thread* worker_thread() { return worker_thread_; }
    151   ViEWrapper* vie() { return vie_wrapper_.get(); }
    152   const VideoFormat& default_codec_format() const {
    153     return default_codec_format_;
    154   }
    155   int GetLastEngineError();
    156   bool FindCodec(const VideoCodec& in);
    157   bool CanSendCodec(const VideoCodec& in, const VideoCodec& current,
    158                     VideoCodec* out);
    159   void RegisterChannel(WebRtcVideoMediaChannel* channel);
    160   void UnregisterChannel(WebRtcVideoMediaChannel* channel);
    161   bool ConvertFromCricketVideoCodec(const VideoCodec& in_codec,
    162                                     webrtc::VideoCodec* out_codec);
    163   // Check whether the supplied trace should be ignored.
    164   bool ShouldIgnoreTrace(const std::string& trace);
    165   int GetNumOfChannels();
    166 
    167   VideoFormat GetStartCaptureFormat() const { return default_codec_format_; }
    168 
    169   talk_base::CpuMonitor* cpu_monitor() { return cpu_monitor_.get(); }
    170 
    171  protected:
    172   // When a video processor registers with the engine.
    173   // SignalMediaFrame will be invoked for every video frame.
    174   // See videoprocessor.h for param reference.
    175   sigslot::signal3<uint32, VideoFrame*, bool*> SignalMediaFrame;
    176 
    177  private:
    178   typedef std::vector<WebRtcVideoMediaChannel*> VideoChannels;
    179   struct VideoCodecPref {
    180     const char* name;
    181     int payload_type;
    182     int pref;
    183   };
    184 
    185   static const VideoCodecPref kVideoCodecPrefs[];
    186   static const VideoFormatPod kVideoFormats[];
    187   static const VideoFormatPod kDefaultVideoFormat;
    188 
    189   void Construct(ViEWrapper* vie_wrapper,
    190                  ViETraceWrapper* tracing,
    191                  WebRtcVoiceEngine* voice_engine,
    192                  talk_base::CpuMonitor* cpu_monitor);
    193   bool SetDefaultCodec(const VideoCodec& codec);
    194   bool RebuildCodecList(const VideoCodec& max_codec);
    195   void SetTraceFilter(int filter);
    196   void SetTraceOptions(const std::string& options);
    197   bool InitVideoEngine();
    198 
    199   // webrtc::TraceCallback implementation.
    200   virtual void Print(webrtc::TraceLevel level, const char* trace, int length);
    201 
    202   // WebRtcVideoEncoderFactory::Observer implementation.
    203   virtual void OnCodecsAvailable();
    204 
    205   talk_base::Thread* worker_thread_;
    206   talk_base::scoped_ptr<ViEWrapper> vie_wrapper_;
    207   bool vie_wrapper_base_initialized_;
    208   talk_base::scoped_ptr<ViETraceWrapper> tracing_;
    209   WebRtcVoiceEngine* voice_engine_;
    210   talk_base::scoped_ptr<webrtc::VideoRender> render_module_;
    211   WebRtcVideoEncoderFactory* encoder_factory_;
    212   WebRtcVideoDecoderFactory* decoder_factory_;
    213   std::vector<VideoCodec> video_codecs_;
    214   std::vector<RtpHeaderExtension> rtp_header_extensions_;
    215   VideoFormat default_codec_format_;
    216 
    217   bool initialized_;
    218   talk_base::CriticalSection channels_crit_;
    219   VideoChannels channels_;
    220 
    221   bool capture_started_;
    222   int local_renderer_w_;
    223   int local_renderer_h_;
    224   VideoRenderer* local_renderer_;
    225 
    226   // Critical section to protect the media processor register/unregister
    227   // while processing a frame
    228   talk_base::CriticalSection signal_media_critical_;
    229 
    230   talk_base::scoped_ptr<talk_base::CpuMonitor> cpu_monitor_;
    231 };
    232 
    233 class WebRtcVideoMediaChannel : public talk_base::MessageHandler,
    234                                 public VideoMediaChannel,
    235                                 public webrtc::Transport {
    236  public:
    237   WebRtcVideoMediaChannel(WebRtcVideoEngine* engine,
    238                           VoiceMediaChannel* voice_channel);
    239   ~WebRtcVideoMediaChannel();
    240   bool Init();
    241 
    242   WebRtcVideoEngine* engine() { return engine_; }
    243   VoiceMediaChannel* voice_channel() { return voice_channel_; }
    244   int video_channel() const { return vie_channel_; }
    245   bool sending() const { return sending_; }
    246 
    247   // VideoMediaChannel implementation
    248   virtual bool SetRecvCodecs(const std::vector<VideoCodec> &codecs);
    249   virtual bool SetSendCodecs(const std::vector<VideoCodec> &codecs);
    250   virtual bool GetSendCodec(VideoCodec* send_codec);
    251   virtual bool SetSendStreamFormat(uint32 ssrc, const VideoFormat& format);
    252   virtual bool SetRender(bool render);
    253   virtual bool SetSend(bool send);
    254 
    255   virtual bool AddSendStream(const StreamParams& sp);
    256   virtual bool RemoveSendStream(uint32 ssrc);
    257   virtual bool AddRecvStream(const StreamParams& sp);
    258   virtual bool RemoveRecvStream(uint32 ssrc);
    259   virtual bool SetRenderer(uint32 ssrc, VideoRenderer* renderer);
    260   virtual bool GetStats(VideoMediaInfo* info);
    261   virtual bool SetCapturer(uint32 ssrc, VideoCapturer* capturer);
    262   virtual bool SendIntraFrame();
    263   virtual bool RequestIntraFrame();
    264 
    265   virtual void OnPacketReceived(talk_base::Buffer* packet);
    266   virtual void OnRtcpReceived(talk_base::Buffer* packet);
    267   virtual void OnReadyToSend(bool ready);
    268   virtual bool MuteStream(uint32 ssrc, bool on);
    269   virtual bool SetRecvRtpHeaderExtensions(
    270       const std::vector<RtpHeaderExtension>& extensions);
    271   virtual bool SetSendRtpHeaderExtensions(
    272       const std::vector<RtpHeaderExtension>& extensions);
    273   virtual bool SetSendBandwidth(bool autobw, int bps);
    274   virtual bool SetOptions(const VideoOptions &options);
    275   virtual bool GetOptions(VideoOptions *options) const {
    276     *options = options_;
    277     return true;
    278   }
    279   virtual void SetInterface(NetworkInterface* iface);
    280   virtual void UpdateAspectRatio(int ratio_w, int ratio_h);
    281 
    282   // Public functions for use by tests and other specialized code.
    283   uint32 send_ssrc() const { return 0; }
    284   bool GetRenderer(uint32 ssrc, VideoRenderer** renderer);
    285   void SendFrame(VideoCapturer* capturer, const VideoFrame* frame);
    286   bool SendFrame(WebRtcVideoChannelSendInfo* channel_info,
    287                  const VideoFrame* frame, bool is_screencast);
    288 
    289   void AdaptAndSendFrame(VideoCapturer* capturer, const VideoFrame* frame);
    290 
    291   // Thunk functions for use with HybridVideoEngine
    292   void OnLocalFrame(VideoCapturer* capturer, const VideoFrame* frame) {
    293     SendFrame(0u, frame, capturer->IsScreencast());
    294   }
    295   void OnLocalFrameFormat(VideoCapturer* capturer, const VideoFormat* format) {
    296   }
    297 
    298   virtual void OnMessage(talk_base::Message* msg);
    299 
    300  protected:
    301   int GetLastEngineError() { return engine()->GetLastEngineError(); }
    302   virtual int SendPacket(int channel, const void* data, int len);
    303   virtual int SendRTCPPacket(int channel, const void* data, int len);
    304 
    305  private:
    306   typedef std::map<uint32, WebRtcVideoChannelRecvInfo*> RecvChannelMap;
    307   typedef std::map<uint32, WebRtcVideoChannelSendInfo*> SendChannelMap;
    308   typedef int (webrtc::ViERTP_RTCP::* ExtensionSetterFunction)(int, bool, int);
    309 
    310   enum MediaDirection { MD_RECV, MD_SEND, MD_SENDRECV };
    311 
    312   // Creates and initializes a ViE channel. When successful |channel_id| will
    313   // contain the new channel's ID. If |receiving| is true |ssrc| is the
    314   // remote ssrc. If |sending| is true the ssrc is local ssrc. If both
    315   // |receiving| and |sending| is true the ssrc must be 0 and the channel will
    316   // be created as a default channel. The ssrc must be different for receive
    317   // channels and it must be different for send channels. If the same SSRC is
    318   // being used for creating channel more than once, this function will fail
    319   // returning false.
    320   bool CreateChannel(uint32 ssrc_key, MediaDirection direction,
    321                      int* channel_id);
    322   bool ConfigureChannel(int channel_id, MediaDirection direction,
    323                         uint32 ssrc_key);
    324   bool ConfigureReceiving(int channel_id, uint32 remote_ssrc_key);
    325   bool ConfigureSending(int channel_id, uint32 local_ssrc_key);
    326   bool SetNackFec(int channel_id, int red_payload_type, int fec_payload_type,
    327                   bool nack_enabled);
    328   bool SetSendCodec(const webrtc::VideoCodec& codec, int min_bitrate,
    329                     int start_bitrate, int max_bitrate);
    330   bool SetSendCodec(WebRtcVideoChannelSendInfo* send_channel,
    331                     const webrtc::VideoCodec& codec, int min_bitrate,
    332                     int start_bitrate, int max_bitrate);
    333   void LogSendCodecChange(const std::string& reason);
    334   // Prepares the channel with channel id |info->channel_id()| to receive all
    335   // codecs in |receive_codecs_| and start receive packets.
    336   bool SetReceiveCodecs(WebRtcVideoChannelRecvInfo* info);
    337   // Returns the channel number that receives the stream with SSRC |ssrc|.
    338   int GetRecvChannelNum(uint32 ssrc);
    339   // Given captured video frame size, checks if we need to reset vie send codec.
    340   // |reset| is set to whether resetting has happened on vie or not.
    341   // Returns false on error.
    342   bool MaybeResetVieSendCodec(WebRtcVideoChannelSendInfo* send_channel,
    343                               int new_width, int new_height, bool is_screencast,
    344                               bool* reset);
    345   // Checks the current bitrate estimate and modifies the start bitrate
    346   // accordingly.
    347   void MaybeChangeStartBitrate(int channel_id, webrtc::VideoCodec* video_codec);
    348   // Helper function for starting the sending of media on all channels or
    349   // |channel_id|. Note that these two function do not change |sending_|.
    350   bool StartSend();
    351   bool StartSend(WebRtcVideoChannelSendInfo* send_channel);
    352   // Helper function for stop the sending of media on all channels or
    353   // |channel_id|. Note that these two function do not change |sending_|.
    354   bool StopSend();
    355   bool StopSend(WebRtcVideoChannelSendInfo* send_channel);
    356   bool SendIntraFrame(int channel_id);
    357 
    358   // Send with one local SSRC. Normal case.
    359   bool IsOneSsrcStream(const StreamParams& sp);
    360 
    361   bool HasReadySendChannels();
    362 
    363   // Send channel key returns the key corresponding to the provided local SSRC
    364   // in |key|. The return value is true upon success.
    365   // If the local ssrc correspond to that of the default channel the key is 0.
    366   // For all other channels the returned key will be the same as the local ssrc.
    367   bool GetSendChannelKey(uint32 local_ssrc, uint32* key);
    368   WebRtcVideoChannelSendInfo* GetSendChannel(VideoCapturer* video_capturer);
    369   WebRtcVideoChannelSendInfo* GetSendChannel(uint32 local_ssrc);
    370   // Creates a new unique key that can be used for inserting a new send channel
    371   // into |send_channels_|
    372   bool CreateSendChannelKey(uint32 local_ssrc, uint32* key);
    373 
    374   bool IsDefaultChannel(int channel_id) const {
    375     return channel_id == vie_channel_;
    376   }
    377   uint32 GetDefaultChannelSsrc();
    378 
    379   bool DeleteSendChannel(uint32 ssrc_key);
    380 
    381   bool InConferenceMode() const {
    382     return options_.conference_mode.GetWithDefaultIfUnset(false);
    383   }
    384   bool RemoveCapturer(uint32 ssrc);
    385 
    386 
    387   talk_base::MessageQueue* worker_thread() { return engine_->worker_thread(); }
    388   void QueueBlackFrame(uint32 ssrc, int64 timestamp, int framerate);
    389   void FlushBlackFrame(uint32 ssrc, int64 timestamp);
    390 
    391   void SetNetworkTransmissionState(bool is_transmitting);
    392 
    393   bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id,
    394                           const RtpHeaderExtension* extension);
    395   bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id,
    396                           const std::vector<RtpHeaderExtension>& extensions,
    397                           const char header_extension_uri[]);
    398 
    399   // Signal when cpu adaptation has no further scope to adapt.
    400   void OnCpuAdaptationUnable();
    401 
    402   // Global state.
    403   WebRtcVideoEngine* engine_;
    404   VoiceMediaChannel* voice_channel_;
    405   int vie_channel_;
    406   bool nack_enabled_;
    407   // Receiver Estimated Max Bitrate
    408   bool remb_enabled_;
    409   VideoOptions options_;
    410 
    411   // Global recv side state.
    412   // Note the default channel (vie_channel_), i.e. the send channel
    413   // corresponding to all the receive channels (this must be done for REMB to
    414   // work properly), resides in both recv_channels_ and send_channels_ with the
    415   // ssrc key 0.
    416   RecvChannelMap recv_channels_;  // Contains all receive channels.
    417   std::vector<webrtc::VideoCodec> receive_codecs_;
    418   bool render_started_;
    419   uint32 first_receive_ssrc_;
    420   std::vector<RtpHeaderExtension> receive_extensions_;
    421 
    422   // Global send side state.
    423   SendChannelMap send_channels_;
    424   talk_base::scoped_ptr<webrtc::VideoCodec> send_codec_;
    425   int send_red_type_;
    426   int send_fec_type_;
    427   int send_min_bitrate_;
    428   int send_start_bitrate_;
    429   int send_max_bitrate_;
    430   bool sending_;
    431   std::vector<RtpHeaderExtension> send_extensions_;
    432 
    433   // The aspect ratio that the channel desires. 0 means there is no desired
    434   // aspect ratio
    435   int ratio_w_;
    436   int ratio_h_;
    437 };
    438 
    439 }  // namespace cricket
    440 
    441 #endif  // TALK_MEDIA_WEBRTCVIDEOENGINE_H_
    442