Home | History | Annotate | Download | only in video 
      1 /*
      2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #ifndef WEBRTC_VIDEO_VIE_CHANNEL_H_
     12 #define WEBRTC_VIDEO_VIE_CHANNEL_H_
     13 
     14 #include <list>
     15 #include <map>
     16 #include <vector>
     17 
     18 #include "webrtc/base/platform_thread.h"
     19 #include "webrtc/base/scoped_ptr.h"
     20 #include "webrtc/base/scoped_ref_ptr.h"
     21 #include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
     22 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
     23 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
     24 #include "webrtc/modules/video_coding/include/video_coding_defines.h"
     25 #include "webrtc/system_wrappers/include/critical_section_wrapper.h"
     26 #include "webrtc/system_wrappers/include/tick_util.h"
     27 #include "webrtc/typedefs.h"
     28 #include "webrtc/video/vie_receiver.h"
     29 #include "webrtc/video/vie_sync_module.h"
     30 
     31 namespace webrtc {
     32 
     33 class CallStatsObserver;
     34 class ChannelStatsObserver;
     35 class Config;
     36 class CriticalSectionWrapper;
     37 class EncodedImageCallback;
     38 class I420FrameCallback;
     39 class IncomingVideoStream;
     40 class PacedSender;
     41 class PacketRouter;
     42 class PayloadRouter;
     43 class ProcessThread;
     44 class ReceiveStatisticsProxy;
     45 class ReportBlockStats;
     46 class RtcpRttStats;
     47 class ViEChannelProtectionCallback;
     48 class ViERTPObserver;
     49 class VideoCodingModule;
     50 class VideoDecoder;
     51 class VideoRenderCallback;
     52 class VoEVideoSync;
     53 
     54 enum StreamType {
     55   kViEStreamTypeNormal = 0,  // Normal media stream
     56   kViEStreamTypeRtx = 1      // Retransmission media stream
     57 };
     58 
     59 class ViEChannel : public VCMFrameTypeCallback,
     60                    public VCMReceiveCallback,
     61                    public VCMReceiveStatisticsCallback,
     62                    public VCMDecoderTimingCallback,
     63                    public VCMPacketRequestCallback,
     64                    public RtpFeedback {
     65  public:
     66   friend class ChannelStatsObserver;
     67   friend class ViEChannelProtectionCallback;
     68 
     69   ViEChannel(uint32_t number_of_cores,
     70              Transport* transport,
     71              ProcessThread* module_process_thread,
     72              RtcpIntraFrameObserver* intra_frame_observer,
     73              RtcpBandwidthObserver* bandwidth_observer,
     74              TransportFeedbackObserver* transport_feedback_observer,
     75              RemoteBitrateEstimator* remote_bitrate_estimator,
     76              RtcpRttStats* rtt_stats,
     77              PacedSender* paced_sender,
     78              PacketRouter* packet_router,
     79              size_t max_rtp_streams,
     80              bool sender);
     81   ~ViEChannel();
     82 
     83   int32_t Init();
     84 
     85   // Sets the encoder to use for the channel. |new_stream| indicates the encoder
     86   // type has changed and we should start a new RTP stream.
     87   int32_t SetSendCodec(const VideoCodec& video_codec, bool new_stream = true);
     88   int32_t SetReceiveCodec(const VideoCodec& video_codec);
     89   // Registers an external decoder.
     90   void RegisterExternalDecoder(const uint8_t pl_type, VideoDecoder* decoder);
     91   int32_t ReceiveCodecStatistics(uint32_t* num_key_frames,
     92                                  uint32_t* num_delta_frames);
     93   uint32_t DiscardedPackets() const;
     94 
     95   // Returns the estimated delay in milliseconds.
     96   int ReceiveDelay() const;
     97 
     98   void SetExpectedRenderDelay(int delay_ms);
     99 
    100   void SetRTCPMode(const RtcpMode rtcp_mode);
    101   void SetProtectionMode(bool enable_nack,
    102                          bool enable_fec,
    103                          int payload_type_red,
    104                          int payload_type_fec);
    105   bool IsSendingFecEnabled();
    106   int SetSenderBufferingMode(int target_delay_ms);
    107   int SetSendTimestampOffsetStatus(bool enable, int id);
    108   int SetReceiveTimestampOffsetStatus(bool enable, int id);
    109   int SetSendAbsoluteSendTimeStatus(bool enable, int id);
    110   int SetReceiveAbsoluteSendTimeStatus(bool enable, int id);
    111   int SetSendVideoRotationStatus(bool enable, int id);
    112   int SetReceiveVideoRotationStatus(bool enable, int id);
    113   int SetSendTransportSequenceNumber(bool enable, int id);
    114   int SetReceiveTransportSequenceNumber(bool enable, int id);
    115   void SetRtcpXrRrtrStatus(bool enable);
    116   void EnableTMMBR(bool enable);
    117 
    118   // Sets SSRC for outgoing stream.
    119   int32_t SetSSRC(const uint32_t SSRC,
    120                   const StreamType usage,
    121                   const unsigned char simulcast_idx);
    122 
    123   // Gets SSRC for outgoing stream number |idx|.
    124   int32_t GetLocalSSRC(uint8_t idx, unsigned int* ssrc);
    125 
    126   // Gets SSRC for the incoming stream.
    127   uint32_t GetRemoteSSRC();
    128 
    129   int SetRtxSendPayloadType(int payload_type, int associated_payload_type);
    130   void SetRtxReceivePayloadType(int payload_type, int associated_payload_type);
    131   // If set to true, the RTX payload type mapping supplied in
    132   // |SetRtxReceivePayloadType| will be used when restoring RTX packets. Without
    133   // it, RTX packets will always be restored to the last non-RTX packet payload
    134   // type received.
    135   void SetUseRtxPayloadMappingOnRestore(bool val);
    136 
    137   void SetRtpStateForSsrc(uint32_t ssrc, const RtpState& rtp_state);
    138   RtpState GetRtpStateForSsrc(uint32_t ssrc);
    139 
    140   // Sets the CName for the outgoing stream on the channel.
    141   int32_t SetRTCPCName(const char* rtcp_cname);
    142 
    143   // Gets the CName of the incoming stream.
    144   int32_t GetRemoteRTCPCName(char rtcp_cname[]);
    145 
    146   // Returns statistics reported by the remote client in an RTCP packet.
    147   // TODO(pbos): Remove this along with VideoSendStream::GetRtt().
    148   int32_t GetSendRtcpStatistics(uint16_t* fraction_lost,
    149                                 uint32_t* cumulative_lost,
    150                                 uint32_t* extended_max,
    151                                 uint32_t* jitter_samples,
    152                                 int64_t* rtt_ms);
    153 
    154   // Called on receipt of RTCP report block from remote side.
    155   void RegisterSendChannelRtcpStatisticsCallback(
    156       RtcpStatisticsCallback* callback);
    157 
    158   // Called on generation of RTCP stats
    159   void RegisterReceiveChannelRtcpStatisticsCallback(
    160       RtcpStatisticsCallback* callback);
    161 
    162   // Gets send statistics for the rtp and rtx stream.
    163   void GetSendStreamDataCounters(StreamDataCounters* rtp_counters,
    164                                  StreamDataCounters* rtx_counters) const;
    165 
    166   // Gets received stream data counters.
    167   void GetReceiveStreamDataCounters(StreamDataCounters* rtp_counters,
    168                                     StreamDataCounters* rtx_counters) const;
    169 
    170   // Called on update of RTP statistics.
    171   void RegisterSendChannelRtpStatisticsCallback(
    172       StreamDataCountersCallback* callback);
    173 
    174   // Called on update of RTP statistics.
    175   void RegisterReceiveChannelRtpStatisticsCallback(
    176       StreamDataCountersCallback* callback);
    177 
    178   void GetSendRtcpPacketTypeCounter(
    179       RtcpPacketTypeCounter* packet_counter) const;
    180 
    181   void GetReceiveRtcpPacketTypeCounter(
    182       RtcpPacketTypeCounter* packet_counter) const;
    183 
    184   void RegisterSendSideDelayObserver(SendSideDelayObserver* observer);
    185 
    186   // Called on any new send bitrate estimate.
    187   void RegisterSendBitrateObserver(BitrateStatisticsObserver* observer);
    188 
    189   // Implements RtpFeedback.
    190   int32_t OnInitializeDecoder(const int8_t payload_type,
    191                               const char payload_name[RTP_PAYLOAD_NAME_SIZE],
    192                               const int frequency,
    193                               const size_t channels,
    194                               const uint32_t rate) override;
    195   void OnIncomingSSRCChanged(const uint32_t ssrc) override;
    196   void OnIncomingCSRCChanged(const uint32_t CSRC, const bool added) override;
    197 
    198   int32_t SetRemoteSSRCType(const StreamType usage, const uint32_t SSRC);
    199 
    200   int32_t StartSend();
    201   int32_t StopSend();
    202   bool Sending();
    203   void StartReceive();
    204   void StopReceive();
    205 
    206   int32_t ReceivedRTPPacket(const void* rtp_packet,
    207                             const size_t rtp_packet_length,
    208                             const PacketTime& packet_time);
    209   int32_t ReceivedRTCPPacket(const void* rtcp_packet,
    210                              const size_t rtcp_packet_length);
    211 
    212   // Sets the maximum transfer unit size for the network link, i.e. including
    213   // IP, UDP and RTP headers.
    214   int32_t SetMTU(uint16_t mtu);
    215 
    216   // Gets the modules used by the channel.
    217   RtpRtcp* rtp_rtcp();
    218   rtc::scoped_refptr<PayloadRouter> send_payload_router();
    219   VCMProtectionCallback* vcm_protection_callback();
    220 
    221 
    222   CallStatsObserver* GetStatsObserver();
    223 
    224   // Implements VCMReceiveCallback.
    225   virtual int32_t FrameToRender(VideoFrame& video_frame);  // NOLINT
    226 
    227   // Implements VCMReceiveCallback.
    228   virtual int32_t ReceivedDecodedReferenceFrame(
    229       const uint64_t picture_id);
    230 
    231   // Implements VCMReceiveCallback.
    232   void OnIncomingPayloadType(int payload_type) override;
    233   void OnDecoderImplementationName(const char* implementation_name) override;
    234 
    235   // Implements VCMReceiveStatisticsCallback.
    236   void OnReceiveRatesUpdated(uint32_t bit_rate, uint32_t frame_rate) override;
    237   void OnDiscardedPacketsUpdated(int discarded_packets) override;
    238   void OnFrameCountsUpdated(const FrameCounts& frame_counts) override;
    239 
    240   // Implements VCMDecoderTimingCallback.
    241   virtual void OnDecoderTiming(int decode_ms,
    242                                int max_decode_ms,
    243                                int current_delay_ms,
    244                                int target_delay_ms,
    245                                int jitter_buffer_ms,
    246                                int min_playout_delay_ms,
    247                                int render_delay_ms);
    248 
    249   // Implements FrameTypeCallback.
    250   virtual int32_t RequestKeyFrame();
    251 
    252   // Implements FrameTypeCallback.
    253   virtual int32_t SliceLossIndicationRequest(
    254       const uint64_t picture_id);
    255 
    256   // Implements VideoPacketRequestCallback.
    257   int32_t ResendPackets(const uint16_t* sequence_numbers,
    258                         uint16_t length) override;
    259 
    260   int32_t SetVoiceChannel(int32_t ve_channel_id,
    261                           VoEVideoSync* ve_sync_interface);
    262   int32_t VoiceChannel();
    263 
    264   // New-style callbacks, used by VideoReceiveStream.
    265   void RegisterPreRenderCallback(I420FrameCallback* pre_render_callback);
    266   void RegisterPreDecodeImageCallback(
    267       EncodedImageCallback* pre_decode_callback);
    268 
    269   void RegisterSendFrameCountObserver(FrameCountObserver* observer);
    270   void RegisterRtcpPacketTypeCounterObserver(
    271       RtcpPacketTypeCounterObserver* observer);
    272   void RegisterReceiveStatisticsProxy(
    273       ReceiveStatisticsProxy* receive_statistics_proxy);
    274   void SetIncomingVideoStream(IncomingVideoStream* incoming_video_stream);
    275 
    276  protected:
    277   static bool ChannelDecodeThreadFunction(void* obj);
    278   bool ChannelDecodeProcess();
    279 
    280   void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms);
    281 
    282   int ProtectionRequest(const FecProtectionParams* delta_fec_params,
    283                         const FecProtectionParams* key_fec_params,
    284                         uint32_t* sent_video_rate_bps,
    285                         uint32_t* sent_nack_rate_bps,
    286                         uint32_t* sent_fec_rate_bps);
    287 
    288  private:
    289   static std::vector<RtpRtcp*> CreateRtpRtcpModules(
    290       bool receiver_only,
    291       ReceiveStatistics* receive_statistics,
    292       Transport* outgoing_transport,
    293       RtcpIntraFrameObserver* intra_frame_callback,
    294       RtcpBandwidthObserver* bandwidth_callback,
    295       TransportFeedbackObserver* transport_feedback_callback,
    296       RtcpRttStats* rtt_stats,
    297       RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
    298       RemoteBitrateEstimator* remote_bitrate_estimator,
    299       RtpPacketSender* paced_sender,
    300       TransportSequenceNumberAllocator* transport_sequence_number_allocator,
    301       BitrateStatisticsObserver* send_bitrate_observer,
    302       FrameCountObserver* send_frame_count_observer,
    303       SendSideDelayObserver* send_side_delay_observer,
    304       size_t num_modules);
    305 
    306   // Assumed to be protected.
    307   void StartDecodeThread();
    308   void StopDecodeThread();
    309 
    310   void ProcessNACKRequest(const bool enable);
    311   // Compute NACK list parameters for the buffering mode.
    312   int GetRequiredNackListSize(int target_delay_ms);
    313   void SetRtxSendStatus(bool enable);
    314 
    315   void UpdateHistograms();
    316 
    317   // ViEChannel exposes methods that allow to modify observers and callbacks
    318   // to be modified. Such an API-style is cumbersome to implement and maintain
    319   // at all the levels when comparing to only setting them at construction. As
    320   // so this class instantiates its children with a wrapper that can be modified
    321   // at a later time.
    322   template <class T>
    323   class RegisterableCallback : public T {
    324    public:
    325     RegisterableCallback()
    326         : critsect_(CriticalSectionWrapper::CreateCriticalSection()),
    327           callback_(NULL) {}
    328 
    329     void Set(T* callback) {
    330       CriticalSectionScoped cs(critsect_.get());
    331       callback_ = callback;
    332     }
    333 
    334    protected:
    335     // Note: this should be implemented with a RW-lock to allow simultaneous
    336     // calls into the callback. However that doesn't seem to be needed for the
    337     // current type of callbacks covered by this class.
    338     rtc::scoped_ptr<CriticalSectionWrapper> critsect_;
    339     T* callback_ GUARDED_BY(critsect_);
    340 
    341    private:
    342     RTC_DISALLOW_COPY_AND_ASSIGN(RegisterableCallback);
    343   };
    344 
    345   class RegisterableBitrateStatisticsObserver:
    346     public RegisterableCallback<BitrateStatisticsObserver> {
    347     virtual void Notify(const BitrateStatistics& total_stats,
    348                         const BitrateStatistics& retransmit_stats,
    349                         uint32_t ssrc) {
    350       CriticalSectionScoped cs(critsect_.get());
    351       if (callback_)
    352         callback_->Notify(total_stats, retransmit_stats, ssrc);
    353     }
    354   } send_bitrate_observer_;
    355 
    356   class RegisterableFrameCountObserver
    357       : public RegisterableCallback<FrameCountObserver> {
    358    public:
    359     virtual void FrameCountUpdated(const FrameCounts& frame_counts,
    360                                    uint32_t ssrc) {
    361       CriticalSectionScoped cs(critsect_.get());
    362       if (callback_)
    363         callback_->FrameCountUpdated(frame_counts, ssrc);
    364     }
    365 
    366    private:
    367   } send_frame_count_observer_;
    368 
    369   class RegisterableSendSideDelayObserver :
    370       public RegisterableCallback<SendSideDelayObserver> {
    371     void SendSideDelayUpdated(int avg_delay_ms,
    372                               int max_delay_ms,
    373                               uint32_t ssrc) override {
    374       CriticalSectionScoped cs(critsect_.get());
    375       if (callback_)
    376         callback_->SendSideDelayUpdated(avg_delay_ms, max_delay_ms, ssrc);
    377     }
    378   } send_side_delay_observer_;
    379 
    380   class RegisterableRtcpPacketTypeCounterObserver
    381       : public RegisterableCallback<RtcpPacketTypeCounterObserver> {
    382    public:
    383     void RtcpPacketTypesCounterUpdated(
    384         uint32_t ssrc,
    385         const RtcpPacketTypeCounter& packet_counter) override {
    386       CriticalSectionScoped cs(critsect_.get());
    387       if (callback_)
    388         callback_->RtcpPacketTypesCounterUpdated(ssrc, packet_counter);
    389       counter_map_[ssrc] = packet_counter;
    390     }
    391 
    392     virtual std::map<uint32_t, RtcpPacketTypeCounter> GetPacketTypeCounterMap()
    393         const {
    394       CriticalSectionScoped cs(critsect_.get());
    395       return counter_map_;
    396     }
    397 
    398    private:
    399     std::map<uint32_t, RtcpPacketTypeCounter> counter_map_
    400         GUARDED_BY(critsect_);
    401   } rtcp_packet_type_counter_observer_;
    402 
    403   const uint32_t number_of_cores_;
    404   const bool sender_;
    405 
    406   ProcessThread* const module_process_thread_;
    407 
    408   // Used for all registered callbacks except rendering.
    409   rtc::scoped_ptr<CriticalSectionWrapper> crit_;
    410 
    411   // Owned modules/classes.
    412   rtc::scoped_refptr<PayloadRouter> send_payload_router_;
    413   rtc::scoped_ptr<ViEChannelProtectionCallback> vcm_protection_callback_;
    414 
    415   VideoCodingModule* const vcm_;
    416   ViEReceiver vie_receiver_;
    417   ViESyncModule vie_sync_;
    418 
    419   // Helper to report call statistics.
    420   rtc::scoped_ptr<ChannelStatsObserver> stats_observer_;
    421 
    422   // Not owned.
    423   ReceiveStatisticsProxy* receive_stats_callback_ GUARDED_BY(crit_);
    424   FrameCounts receive_frame_counts_ GUARDED_BY(crit_);
    425   IncomingVideoStream* incoming_video_stream_ GUARDED_BY(crit_);
    426   RtcpIntraFrameObserver* const intra_frame_observer_;
    427   RtcpRttStats* const rtt_stats_;
    428   PacedSender* const paced_sender_;
    429   PacketRouter* const packet_router_;
    430 
    431   const rtc::scoped_ptr<RtcpBandwidthObserver> bandwidth_observer_;
    432   TransportFeedbackObserver* const transport_feedback_observer_;
    433 
    434   rtc::PlatformThread decode_thread_;
    435 
    436   int nack_history_size_sender_;
    437   int max_nack_reordering_threshold_;
    438   I420FrameCallback* pre_render_callback_ GUARDED_BY(crit_);
    439 
    440   const rtc::scoped_ptr<ReportBlockStats> report_block_stats_sender_;
    441 
    442   int64_t time_of_first_rtt_ms_ GUARDED_BY(crit_);
    443   int64_t rtt_sum_ms_ GUARDED_BY(crit_);
    444   int64_t last_rtt_ms_ GUARDED_BY(crit_);
    445   size_t num_rtts_ GUARDED_BY(crit_);
    446 
    447   // RtpRtcp modules, declared last as they use other members on construction.
    448   const std::vector<RtpRtcp*> rtp_rtcp_modules_;
    449   size_t num_active_rtp_rtcp_modules_ GUARDED_BY(crit_);
    450 };
    451 
    452 }  // namespace webrtc
    453 
    454 #endif  // WEBRTC_VIDEO_VIE_CHANNEL_H_
    455