Home | History | Annotate | Download | only in webrtc 
      1 /*
      2  * libjingle
      3  * Copyright 2014 Google Inc.
      4  *
      5  * Redistribution and use in source and binary forms, with or without
      6  * modification, are permitted provided that the following conditions are met:
      7  *
      8  *  1. Redistributions of source code must retain the above copyright notice,
      9  *     this list of conditions and the following disclaimer.
     10  *  2. Redistributions in binary form must reproduce the above copyright notice,
     11  *     this list of conditions and the following disclaimer in the documentation
     12  *     and/or other materials provided with the distribution.
     13  *  3. The name of the author may not be used to endorse or promote products
     14  *     derived from this software without specific prior written permission.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
     17  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
     18  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
     19  * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     20  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
     21  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
     22  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
     23  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
     24  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
     25  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26  */
     27 
     28 #ifndef TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_
     29 #define TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_
     30 
     31 #include <map>
     32 #include <string>
     33 #include <vector>
     34 
     35 #include "talk/media/base/mediaengine.h"
     36 #include "talk/media/webrtc/webrtcvideochannelfactory.h"
     37 #include "talk/media/webrtc/webrtcvideodecoderfactory.h"
     38 #include "talk/media/webrtc/webrtcvideoencoderfactory.h"
     39 #include "webrtc/base/cpumonitor.h"
     40 #include "webrtc/base/scoped_ptr.h"
     41 #include "webrtc/base/thread_annotations.h"
     42 #include "webrtc/common_video/interface/i420_video_frame.h"
     43 #include "webrtc/transport.h"
     44 #include "webrtc/video_receive_stream.h"
     45 #include "webrtc/video_renderer.h"
     46 #include "webrtc/video_send_stream.h"
     47 
     48 namespace webrtc {
     49 class Call;
     50 class VideoCaptureModule;
     51 class VideoDecoder;
     52 class VideoEncoder;
     53 class VideoRender;
     54 class VideoSendStreamInput;
     55 class VideoReceiveStream;
     56 }
     57 
     58 namespace rtc {
     59 class CpuMonitor;
     60 class Thread;
     61 }  // namespace rtc
     62 
     63 namespace cricket {
     64 
     65 class VideoCapturer;
     66 class VideoFrame;
     67 class VideoProcessor;
     68 class VideoRenderer;
     69 class VoiceMediaChannel;
     70 class WebRtcDecoderObserver;
     71 class WebRtcEncoderObserver;
     72 class WebRtcLocalStreamInfo;
     73 class WebRtcRenderAdapter;
     74 class WebRtcVideoChannel2;
     75 class WebRtcVideoChannelRecvInfo;
     76 class WebRtcVideoChannelSendInfo;
     77 class WebRtcVoiceEngine;
     78 
     79 struct CapturedFrame;
     80 struct Device;
     81 
     82 class WebRtcVideoEngine2;
     83 class WebRtcVideoChannel2;
     84 class WebRtcVideoRenderer;
     85 
     86 class UnsignalledSsrcHandler {
     87  public:
     88   enum Action {
     89     kDropPacket,
     90     kDeliverPacket,
     91   };
     92   virtual Action OnUnsignalledSsrc(VideoMediaChannel* engine,
     93                                    uint32_t ssrc) = 0;
     94 };
     95 
     96 // TODO(pbos): Remove, use external handlers only.
     97 class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler {
     98  public:
     99   DefaultUnsignalledSsrcHandler();
    100   virtual Action OnUnsignalledSsrc(VideoMediaChannel* engine,
    101                                    uint32_t ssrc) OVERRIDE;
    102 
    103   VideoRenderer* GetDefaultRenderer() const;
    104   void SetDefaultRenderer(VideoMediaChannel* channel, VideoRenderer* renderer);
    105 
    106  private:
    107   uint32_t default_recv_ssrc_;
    108   VideoRenderer* default_renderer_;
    109 };
    110 
    111 class WebRtcVideoEncoderFactory2 {
    112  public:
    113   virtual ~WebRtcVideoEncoderFactory2();
    114   virtual std::vector<webrtc::VideoStream> CreateVideoStreams(
    115       const VideoCodec& codec,
    116       const VideoOptions& options,
    117       size_t num_streams);
    118 
    119   virtual webrtc::VideoEncoder* CreateVideoEncoder(
    120       const VideoCodec& codec,
    121       const VideoOptions& options);
    122 
    123   virtual void* CreateVideoEncoderSettings(const VideoCodec& codec,
    124                                            const VideoOptions& options);
    125 
    126   virtual void DestroyVideoEncoderSettings(const VideoCodec& codec,
    127                                            void* encoder_settings);
    128 
    129   virtual bool SupportsCodec(const cricket::VideoCodec& codec);
    130 };
    131 
    132 // WebRtcVideoEngine2 is used for the new native WebRTC Video API (webrtc:1667).
    133 class WebRtcVideoEngine2 : public sigslot::has_slots<>,
    134                            public WebRtcVideoEncoderFactory::Observer {
    135  public:
    136   // Creates the WebRtcVideoEngine2 with internal VideoCaptureModule.
    137   WebRtcVideoEngine2();
    138   virtual ~WebRtcVideoEngine2();
    139 
    140   // Use a custom WebRtcVideoChannelFactory (for testing purposes).
    141   void SetChannelFactory(WebRtcVideoChannelFactory* channel_factory);
    142 
    143   // Basic video engine implementation.
    144   bool Init(rtc::Thread* worker_thread);
    145   void Terminate();
    146 
    147   int GetCapabilities();
    148   bool SetDefaultEncoderConfig(const VideoEncoderConfig& config);
    149   VideoEncoderConfig GetDefaultEncoderConfig() const;
    150 
    151   WebRtcVideoChannel2* CreateChannel(VoiceMediaChannel* voice_channel);
    152 
    153   const std::vector<VideoCodec>& codecs() const;
    154   const std::vector<RtpHeaderExtension>& rtp_header_extensions() const;
    155   void SetLogging(int min_sev, const char* filter);
    156 
    157   // Set a WebRtcVideoDecoderFactory for external decoding. Video engine does
    158   // not take the ownership of |decoder_factory|. The caller needs to make sure
    159   // that |decoder_factory| outlives the video engine.
    160   void SetExternalDecoderFactory(WebRtcVideoDecoderFactory* decoder_factory);
    161   // Set a WebRtcVideoEncoderFactory for external encoding. Video engine does
    162   // not take the ownership of |encoder_factory|. The caller needs to make sure
    163   // that |encoder_factory| outlives the video engine.
    164   virtual void SetExternalEncoderFactory(
    165       WebRtcVideoEncoderFactory* encoder_factory);
    166 
    167   bool EnableTimedRender();
    168   // This is currently ignored.
    169   sigslot::repeater2<VideoCapturer*, CaptureState> SignalCaptureStateChange;
    170 
    171   // Set the VoiceEngine for A/V sync. This can only be called before Init.
    172   bool SetVoiceEngine(WebRtcVoiceEngine* voice_engine);
    173 
    174   bool FindCodec(const VideoCodec& in);
    175   bool CanSendCodec(const VideoCodec& in,
    176                     const VideoCodec& current,
    177                     VideoCodec* out);
    178   // Check whether the supplied trace should be ignored.
    179   bool ShouldIgnoreTrace(const std::string& trace);
    180 
    181   VideoFormat GetStartCaptureFormat() const { return default_codec_format_; }
    182 
    183   rtc::CpuMonitor* cpu_monitor() { return cpu_monitor_.get(); }
    184 
    185   virtual WebRtcVideoEncoderFactory2* GetVideoEncoderFactory();
    186 
    187  private:
    188   virtual void OnCodecsAvailable() OVERRIDE;
    189 
    190   rtc::Thread* worker_thread_;
    191   WebRtcVoiceEngine* voice_engine_;
    192   std::vector<VideoCodec> video_codecs_;
    193   std::vector<RtpHeaderExtension> rtp_header_extensions_;
    194   VideoFormat default_codec_format_;
    195 
    196   bool initialized_;
    197 
    198   // Critical section to protect the media processor register/unregister
    199   // while processing a frame
    200   rtc::CriticalSection signal_media_critical_;
    201 
    202   rtc::scoped_ptr<rtc::CpuMonitor> cpu_monitor_;
    203   WebRtcVideoChannelFactory* channel_factory_;
    204   WebRtcVideoEncoderFactory2 default_video_encoder_factory_;
    205 
    206   WebRtcVideoDecoderFactory* external_decoder_factory_;
    207   WebRtcVideoEncoderFactory* external_encoder_factory_;
    208 };
    209 
    210 class WebRtcVideoChannel2 : public rtc::MessageHandler,
    211                             public VideoMediaChannel,
    212                             public webrtc::newapi::Transport {
    213  public:
    214   WebRtcVideoChannel2(WebRtcVideoEngine2* engine,
    215                       VoiceMediaChannel* voice_channel,
    216                       WebRtcVideoEncoderFactory2* encoder_factory);
    217   // For testing purposes insert a pre-constructed call to verify that
    218   // WebRtcVideoChannel2 calls the correct corresponding methods.
    219   WebRtcVideoChannel2(webrtc::Call* call,
    220                       WebRtcVideoEngine2* engine,
    221                       WebRtcVideoEncoderFactory2* encoder_factory);
    222   ~WebRtcVideoChannel2();
    223   bool Init();
    224 
    225   // VideoMediaChannel implementation
    226   virtual bool SetRecvCodecs(const std::vector<VideoCodec>& codecs) OVERRIDE;
    227   virtual bool SetSendCodecs(const std::vector<VideoCodec>& codecs) OVERRIDE;
    228   virtual bool GetSendCodec(VideoCodec* send_codec) OVERRIDE;
    229   virtual bool SetSendStreamFormat(uint32 ssrc,
    230                                    const VideoFormat& format) OVERRIDE;
    231   virtual bool SetRender(bool render) OVERRIDE;
    232   virtual bool SetSend(bool send) OVERRIDE;
    233 
    234   virtual bool AddSendStream(const StreamParams& sp) OVERRIDE;
    235   virtual bool RemoveSendStream(uint32 ssrc) OVERRIDE;
    236   virtual bool AddRecvStream(const StreamParams& sp) OVERRIDE;
    237   virtual bool RemoveRecvStream(uint32 ssrc) OVERRIDE;
    238   virtual bool SetRenderer(uint32 ssrc, VideoRenderer* renderer) OVERRIDE;
    239   virtual bool GetStats(const StatsOptions& options,
    240                         VideoMediaInfo* info) OVERRIDE;
    241   virtual bool SetCapturer(uint32 ssrc, VideoCapturer* capturer) OVERRIDE;
    242   virtual bool SendIntraFrame() OVERRIDE;
    243   virtual bool RequestIntraFrame() OVERRIDE;
    244 
    245   virtual void OnPacketReceived(rtc::Buffer* packet,
    246                                 const rtc::PacketTime& packet_time)
    247       OVERRIDE;
    248   virtual void OnRtcpReceived(rtc::Buffer* packet,
    249                               const rtc::PacketTime& packet_time)
    250       OVERRIDE;
    251   virtual void OnReadyToSend(bool ready) OVERRIDE;
    252   virtual bool MuteStream(uint32 ssrc, bool mute) OVERRIDE;
    253 
    254   // Set send/receive RTP header extensions. This must be done before creating
    255   // streams as it only has effect on future streams.
    256   virtual bool SetRecvRtpHeaderExtensions(
    257       const std::vector<RtpHeaderExtension>& extensions) OVERRIDE;
    258   virtual bool SetSendRtpHeaderExtensions(
    259       const std::vector<RtpHeaderExtension>& extensions) OVERRIDE;
    260   virtual bool SetStartSendBandwidth(int bps) OVERRIDE;
    261   virtual bool SetMaxSendBandwidth(int bps) OVERRIDE;
    262   virtual bool SetOptions(const VideoOptions& options) OVERRIDE;
    263   virtual bool GetOptions(VideoOptions* options) const OVERRIDE {
    264     *options = options_;
    265     return true;
    266   }
    267   virtual void SetInterface(NetworkInterface* iface) OVERRIDE;
    268   virtual void UpdateAspectRatio(int ratio_w, int ratio_h) OVERRIDE;
    269 
    270   virtual void OnMessage(rtc::Message* msg) OVERRIDE;
    271 
    272   // Implemented for VideoMediaChannelTest.
    273   bool sending() const { return sending_; }
    274   uint32 GetDefaultSendChannelSsrc() { return default_send_ssrc_; }
    275   bool GetRenderer(uint32 ssrc, VideoRenderer** renderer);
    276 
    277  private:
    278   void ConfigureReceiverRtp(webrtc::VideoReceiveStream::Config* config,
    279                             const StreamParams& sp) const;
    280 
    281   struct VideoCodecSettings {
    282     VideoCodecSettings();
    283 
    284     VideoCodec codec;
    285     webrtc::FecConfig fec;
    286     int rtx_payload_type;
    287   };
    288 
    289   // Wrapper for the sender part, this is where the capturer is connected and
    290   // frames are then converted from cricket frames to webrtc frames.
    291   class WebRtcVideoSendStream : public sigslot::has_slots<> {
    292    public:
    293     WebRtcVideoSendStream(
    294         webrtc::Call* call,
    295         WebRtcVideoEncoderFactory2* encoder_factory,
    296         const VideoOptions& options,
    297         const Settable<VideoCodecSettings>& codec_settings,
    298         const StreamParams& sp,
    299         const std::vector<webrtc::RtpExtension>& rtp_extensions);
    300 
    301     ~WebRtcVideoSendStream();
    302     void SetOptions(const VideoOptions& options);
    303     void SetCodec(const VideoCodecSettings& codec);
    304     void SetRtpExtensions(
    305         const std::vector<webrtc::RtpExtension>& rtp_extensions);
    306 
    307     void InputFrame(VideoCapturer* capturer, const VideoFrame* frame);
    308     bool SetCapturer(VideoCapturer* capturer);
    309     bool SetVideoFormat(const VideoFormat& format);
    310     void MuteStream(bool mute);
    311     bool DisconnectCapturer();
    312 
    313     void Start();
    314     void Stop();
    315 
    316     VideoSenderInfo GetVideoSenderInfo();
    317 
    318    private:
    319     // Parameters needed to reconstruct the underlying stream.
    320     // webrtc::VideoSendStream doesn't support setting a lot of options on the
    321     // fly, so when those need to be changed we tear down and reconstruct with
    322     // similar parameters depending on which options changed etc.
    323     struct VideoSendStreamParameters {
    324       VideoSendStreamParameters(
    325           const webrtc::VideoSendStream::Config& config,
    326           const VideoOptions& options,
    327           const Settable<VideoCodecSettings>& codec_settings);
    328       webrtc::VideoSendStream::Config config;
    329       VideoOptions options;
    330       Settable<VideoCodecSettings> codec_settings;
    331       // Sent resolutions + bitrates etc. by the underlying VideoSendStream,
    332       // typically changes when setting a new resolution or reconfiguring
    333       // bitrates.
    334       webrtc::VideoEncoderConfig encoder_config;
    335     };
    336 
    337     void SetCodecAndOptions(const VideoCodecSettings& codec,
    338                             const VideoOptions& options)
    339         EXCLUSIVE_LOCKS_REQUIRED(lock_);
    340     void RecreateWebRtcStream() EXCLUSIVE_LOCKS_REQUIRED(lock_);
    341     // When |override_max| is false constrain width/height to codec dimensions.
    342     void SetDimensions(int width, int height, bool override_max)
    343         EXCLUSIVE_LOCKS_REQUIRED(lock_);
    344 
    345     webrtc::Call* const call_;
    346     WebRtcVideoEncoderFactory2* const encoder_factory_;
    347 
    348     rtc::CriticalSection lock_;
    349     webrtc::VideoSendStream* stream_ GUARDED_BY(lock_);
    350     VideoSendStreamParameters parameters_ GUARDED_BY(lock_);
    351 
    352     VideoCapturer* capturer_ GUARDED_BY(lock_);
    353     bool sending_ GUARDED_BY(lock_);
    354     bool muted_ GUARDED_BY(lock_);
    355     VideoFormat format_ GUARDED_BY(lock_);
    356 
    357     rtc::CriticalSection frame_lock_;
    358     webrtc::I420VideoFrame video_frame_ GUARDED_BY(frame_lock_);
    359   };
    360 
    361   // Wrapper for the receiver part, contains configs etc. that are needed to
    362   // reconstruct the underlying VideoReceiveStream. Also serves as a wrapper
    363   // between webrtc::VideoRenderer and cricket::VideoRenderer.
    364   class WebRtcVideoReceiveStream : public webrtc::VideoRenderer {
    365    public:
    366     WebRtcVideoReceiveStream(
    367         webrtc::Call*,
    368         const webrtc::VideoReceiveStream::Config& config,
    369         const std::vector<VideoCodecSettings>& recv_codecs);
    370     ~WebRtcVideoReceiveStream();
    371 
    372     void SetRecvCodecs(const std::vector<VideoCodecSettings>& recv_codecs);
    373     void SetRtpExtensions(const std::vector<webrtc::RtpExtension>& extensions);
    374 
    375     virtual void RenderFrame(const webrtc::I420VideoFrame& frame,
    376                              int time_to_render_ms) OVERRIDE;
    377 
    378     void SetRenderer(cricket::VideoRenderer* renderer);
    379     cricket::VideoRenderer* GetRenderer();
    380 
    381     VideoReceiverInfo GetVideoReceiverInfo();
    382 
    383    private:
    384     void SetSize(int width, int height);
    385     void RecreateWebRtcStream();
    386 
    387     webrtc::Call* const call_;
    388 
    389     webrtc::VideoReceiveStream* stream_;
    390     webrtc::VideoReceiveStream::Config config_;
    391 
    392     rtc::CriticalSection renderer_lock_;
    393     cricket::VideoRenderer* renderer_ GUARDED_BY(renderer_lock_);
    394     int last_width_ GUARDED_BY(renderer_lock_);
    395     int last_height_ GUARDED_BY(renderer_lock_);
    396   };
    397 
    398   void Construct(webrtc::Call* call, WebRtcVideoEngine2* engine);
    399   void SetDefaultOptions();
    400 
    401   virtual bool SendRtp(const uint8_t* data, size_t len) OVERRIDE;
    402   virtual bool SendRtcp(const uint8_t* data, size_t len) OVERRIDE;
    403 
    404   void StartAllSendStreams();
    405   void StopAllSendStreams();
    406 
    407   static std::vector<VideoCodecSettings> MapCodecs(
    408       const std::vector<VideoCodec>& codecs);
    409   std::vector<VideoCodecSettings> FilterSupportedCodecs(
    410       const std::vector<VideoCodecSettings>& mapped_codecs);
    411 
    412   void FillSenderStats(VideoMediaInfo* info);
    413   void FillReceiverStats(VideoMediaInfo* info);
    414   void FillBandwidthEstimationStats(VideoMediaInfo* info);
    415 
    416   uint32_t rtcp_receiver_report_ssrc_;
    417   bool sending_;
    418   rtc::scoped_ptr<webrtc::Call> call_;
    419   uint32_t default_send_ssrc_;
    420 
    421   DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_;
    422   UnsignalledSsrcHandler* const unsignalled_ssrc_handler_;
    423 
    424   // Using primary-ssrc (first ssrc) as key.
    425   std::map<uint32, WebRtcVideoSendStream*> send_streams_;
    426   std::map<uint32, WebRtcVideoReceiveStream*> receive_streams_;
    427 
    428   Settable<VideoCodecSettings> send_codec_;
    429   std::vector<webrtc::RtpExtension> send_rtp_extensions_;
    430 
    431   WebRtcVideoEncoderFactory2* const encoder_factory_;
    432   std::vector<VideoCodecSettings> recv_codecs_;
    433   std::vector<webrtc::RtpExtension> recv_rtp_extensions_;
    434   VideoOptions options_;
    435 };
    436 
    437 }  // namespace cricket
    438 
    439 #endif  // TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_
    440