Home | History | Annotate | Download | only in video 
      1 /*
      2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #ifndef WEBRTC_VIDEO_VIE_ENCODER_H_
     12 #define WEBRTC_VIDEO_VIE_ENCODER_H_
     13 
     14 #include <map>
     15 #include <vector>
     16 
     17 #include "webrtc/base/scoped_ptr.h"
     18 #include "webrtc/base/scoped_ref_ptr.h"
     19 #include "webrtc/base/thread_annotations.h"
     20 #include "webrtc/call/bitrate_allocator.h"
     21 #include "webrtc/common_types.h"
     22 #include "webrtc/frame_callback.h"
     23 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
     24 #include "webrtc/modules/video_coding/include/video_coding_defines.h"
     25 #include "webrtc/modules/video_processing/include/video_processing.h"
     26 #include "webrtc/typedefs.h"
     27 #include "webrtc/video/video_capture_input.h"
     28 
     29 namespace webrtc {
     30 
     31 class BitrateAllocator;
     32 class BitrateObserver;
     33 class Config;
     34 class CriticalSectionWrapper;
     35 class EncodedImageCallback;
     36 class PacedSender;
     37 class PayloadRouter;
     38 class ProcessThread;
     39 class QMVideoSettingsCallback;
     40 class SendStatisticsProxy;
     41 class ViEBitrateObserver;
     42 class ViEEffectFilter;
     43 class VideoCodingModule;
     44 
     45 class ViEEncoder : public RtcpIntraFrameObserver,
     46                    public VideoEncoderRateObserver,
     47                    public VCMPacketizationCallback,
     48                    public VCMSendStatisticsCallback,
     49                    public VideoCaptureCallback {
     50  public:
     51   friend class ViEBitrateObserver;
     52 
     53   ViEEncoder(uint32_t number_of_cores,
     54              ProcessThread* module_process_thread,
     55              SendStatisticsProxy* stats_proxy,
     56              I420FrameCallback* pre_encode_callback,
     57              PacedSender* pacer,
     58              BitrateAllocator* bitrate_allocator);
     59   ~ViEEncoder();
     60 
     61   bool Init();
     62 
     63   // This function is assumed to be called before any frames are delivered and
     64   // only once.
     65   // Ideally this would be done in Init, but the dependencies between ViEEncoder
     66   // and ViEChannel makes it really hard to do in a good way.
     67   void StartThreadsAndSetSharedMembers(
     68       rtc::scoped_refptr<PayloadRouter> send_payload_router,
     69       VCMProtectionCallback* vcm_protection_callback);
     70 
     71   // This function must be called before the corresponding ViEChannel is
     72   // deleted.
     73   void StopThreadsAndRemoveSharedMembers();
     74 
     75   void SetNetworkTransmissionState(bool is_transmitting);
     76 
     77   // Returns the id of the owning channel.
     78   int Owner() const;
     79 
     80   // Drops incoming packets before they get to the encoder.
     81   void Pause();
     82   void Restart();
     83 
     84   // Codec settings.
     85   int32_t RegisterExternalEncoder(VideoEncoder* encoder,
     86                                   uint8_t pl_type,
     87                                   bool internal_source);
     88   int32_t DeRegisterExternalEncoder(uint8_t pl_type);
     89   int32_t SetEncoder(const VideoCodec& video_codec);
     90 
     91   // Implementing VideoCaptureCallback.
     92   void DeliverFrame(VideoFrame video_frame) override;
     93 
     94   int32_t SendKeyFrame();
     95 
     96   uint32_t LastObservedBitrateBps() const;
     97   int CodecTargetBitrate(uint32_t* bitrate) const;
     98   // Loss protection. Must be called before SetEncoder() to have max packet size
     99   // updated according to protection.
    100   // TODO(pbos): Set protection method on construction or extract vcm_ outside
    101   // this class and set it on construction there.
    102   void SetProtectionMethod(bool nack, bool fec);
    103 
    104   // Buffering mode.
    105   void SetSenderBufferingMode(int target_delay_ms);
    106 
    107   // Implements VideoEncoderRateObserver.
    108   void OnSetRates(uint32_t bitrate_bps, int framerate) override;
    109 
    110   // Implements VCMPacketizationCallback.
    111   int32_t SendData(uint8_t payload_type,
    112                    const EncodedImage& encoded_image,
    113                    const RTPFragmentationHeader& fragmentation_header,
    114                    const RTPVideoHeader* rtp_video_hdr) override;
    115   void OnEncoderImplementationName(const char* implementation_name) override;
    116 
    117   // Implements VideoSendStatisticsCallback.
    118   int32_t SendStatistics(const uint32_t bit_rate,
    119                          const uint32_t frame_rate) override;
    120 
    121   // Implements RtcpIntraFrameObserver.
    122   void OnReceivedIntraFrameRequest(uint32_t ssrc) override;
    123   void OnReceivedSLI(uint32_t ssrc, uint8_t picture_id) override;
    124   void OnReceivedRPSI(uint32_t ssrc, uint64_t picture_id) override;
    125   void OnLocalSsrcChanged(uint32_t old_ssrc, uint32_t new_ssrc) override;
    126 
    127   // Sets SSRCs for all streams.
    128   void SetSsrcs(const std::vector<uint32_t>& ssrcs);
    129 
    130   void SetMinTransmitBitrate(int min_transmit_bitrate_kbps);
    131 
    132   // Lets the sender suspend video when the rate drops below
    133   // |threshold_bps|, and turns back on when the rate goes back up above
    134   // |threshold_bps| + |window_bps|.
    135   void SuspendBelowMinBitrate();
    136 
    137   // New-style callbacks, used by VideoSendStream.
    138   void RegisterPostEncodeImageCallback(
    139         EncodedImageCallback* post_encode_callback);
    140 
    141   int GetPaddingNeededBps() const;
    142 
    143  protected:
    144   // Called by BitrateObserver.
    145   void OnNetworkChanged(uint32_t bitrate_bps,
    146                         uint8_t fraction_lost,
    147                         int64_t round_trip_time_ms);
    148 
    149  private:
    150   bool EncoderPaused() const EXCLUSIVE_LOCKS_REQUIRED(data_cs_);
    151   void TraceFrameDropStart() EXCLUSIVE_LOCKS_REQUIRED(data_cs_);
    152   void TraceFrameDropEnd() EXCLUSIVE_LOCKS_REQUIRED(data_cs_);
    153 
    154   const uint32_t number_of_cores_;
    155 
    156   const rtc::scoped_ptr<VideoProcessing> vp_;
    157   const rtc::scoped_ptr<QMVideoSettingsCallback> qm_callback_;
    158   const rtc::scoped_ptr<VideoCodingModule> vcm_;
    159   rtc::scoped_refptr<PayloadRouter> send_payload_router_;
    160 
    161   rtc::scoped_ptr<CriticalSectionWrapper> data_cs_;
    162   rtc::scoped_ptr<BitrateObserver> bitrate_observer_;
    163 
    164   SendStatisticsProxy* const stats_proxy_;
    165   I420FrameCallback* const pre_encode_callback_;
    166   PacedSender* const pacer_;
    167   BitrateAllocator* const bitrate_allocator_;
    168 
    169   // The time we last received an input frame or encoded frame. This is used to
    170   // track when video is stopped long enough that we also want to stop sending
    171   // padding.
    172   int64_t time_of_last_frame_activity_ms_ GUARDED_BY(data_cs_);
    173   VideoCodec encoder_config_ GUARDED_BY(data_cs_);
    174   int min_transmit_bitrate_kbps_ GUARDED_BY(data_cs_);
    175   uint32_t last_observed_bitrate_bps_ GUARDED_BY(data_cs_);
    176   int target_delay_ms_ GUARDED_BY(data_cs_);
    177   bool network_is_transmitting_ GUARDED_BY(data_cs_);
    178   bool encoder_paused_ GUARDED_BY(data_cs_);
    179   bool encoder_paused_and_dropped_frame_ GUARDED_BY(data_cs_);
    180   std::map<unsigned int, int64_t> time_last_intra_request_ms_
    181       GUARDED_BY(data_cs_);
    182 
    183   ProcessThread* module_process_thread_;
    184 
    185   bool has_received_sli_ GUARDED_BY(data_cs_);
    186   uint8_t picture_id_sli_ GUARDED_BY(data_cs_);
    187   bool has_received_rpsi_ GUARDED_BY(data_cs_);
    188   uint64_t picture_id_rpsi_ GUARDED_BY(data_cs_);
    189   std::map<uint32_t, int> ssrc_streams_ GUARDED_BY(data_cs_);
    190 
    191   bool video_suspended_ GUARDED_BY(data_cs_);
    192 };
    193 
    194 }  // namespace webrtc
    195 
    196 #endif  // WEBRTC_VIDEO_VIE_ENCODER_H_
    197