Home | History | Annotate | Download | only in video_coding 
      1 /*
      2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #ifndef WEBRTC_MODULES_VIDEO_CODING_JITTER_BUFFER_H_
     12 #define WEBRTC_MODULES_VIDEO_CODING_JITTER_BUFFER_H_
     13 
     14 #include <list>
     15 #include <map>
     16 #include <set>
     17 #include <vector>
     18 
     19 #include "webrtc/base/constructormagic.h"
     20 #include "webrtc/base/thread_annotations.h"
     21 #include "webrtc/modules/include/module_common_types.h"
     22 #include "webrtc/modules/video_coding/include/video_coding.h"
     23 #include "webrtc/modules/video_coding/include/video_coding_defines.h"
     24 #include "webrtc/modules/video_coding/decoding_state.h"
     25 #include "webrtc/modules/video_coding/inter_frame_delay.h"
     26 #include "webrtc/modules/video_coding/jitter_buffer_common.h"
     27 #include "webrtc/modules/video_coding/jitter_estimator.h"
     28 #include "webrtc/system_wrappers/include/critical_section_wrapper.h"
     29 #include "webrtc/typedefs.h"
     30 
     31 namespace webrtc {
     32 
     33 enum VCMNackMode { kNack, kNoNack };
     34 
     35 // forward declarations
     36 class Clock;
     37 class EventFactory;
     38 class EventWrapper;
     39 class VCMFrameBuffer;
     40 class VCMPacket;
     41 class VCMEncodedFrame;
     42 
     43 typedef std::list<VCMFrameBuffer*> UnorderedFrameList;
     44 
     45 struct VCMJitterSample {
     46   VCMJitterSample() : timestamp(0), frame_size(0), latest_packet_time(-1) {}
     47   uint32_t timestamp;
     48   uint32_t frame_size;
     49   int64_t latest_packet_time;
     50 };
     51 
     52 class TimestampLessThan {
     53  public:
     54   bool operator()(uint32_t timestamp1, uint32_t timestamp2) const {
     55     return IsNewerTimestamp(timestamp2, timestamp1);
     56   }
     57 };
     58 
     59 class FrameList
     60     : public std::map<uint32_t, VCMFrameBuffer*, TimestampLessThan> {
     61  public:
     62   void InsertFrame(VCMFrameBuffer* frame);
     63   VCMFrameBuffer* PopFrame(uint32_t timestamp);
     64   VCMFrameBuffer* Front() const;
     65   VCMFrameBuffer* Back() const;
     66   int RecycleFramesUntilKeyFrame(FrameList::iterator* key_frame_it,
     67                                  UnorderedFrameList* free_frames);
     68   void CleanUpOldOrEmptyFrames(VCMDecodingState* decoding_state,
     69                                UnorderedFrameList* free_frames);
     70   void Reset(UnorderedFrameList* free_frames);
     71 };
     72 
     73 class Vp9SsMap {
     74  public:
     75   typedef std::map<uint32_t, GofInfoVP9, TimestampLessThan> SsMap;
     76   bool Insert(const VCMPacket& packet);
     77   void Reset();
     78 
     79   // Removes SS data that are older than |timestamp|.
     80   // The |timestamp| should be an old timestamp, i.e. packets with older
     81   // timestamps should no longer be inserted.
     82   void RemoveOld(uint32_t timestamp);
     83 
     84   bool UpdatePacket(VCMPacket* packet);
     85   void UpdateFrames(FrameList* frames);
     86 
     87   // Public for testing.
     88   // Returns an iterator to the corresponding SS data for the input |timestamp|.
     89   bool Find(uint32_t timestamp, SsMap::iterator* it);
     90 
     91  private:
     92   // These two functions are called by RemoveOld.
     93   // Checks if it is time to do a clean up (done each kSsCleanupIntervalSec).
     94   bool TimeForCleanup(uint32_t timestamp) const;
     95 
     96   // Advances the oldest SS data to handle timestamp wrap in cases where SS data
     97   // are received very seldom (e.g. only once in beginning, second when
     98   // IsNewerTimestamp is not true).
     99   void AdvanceFront(uint32_t timestamp);
    100 
    101   SsMap ss_map_;
    102 };
    103 
    104 class VCMJitterBuffer {
    105  public:
    106   VCMJitterBuffer(Clock* clock, rtc::scoped_ptr<EventWrapper> event);
    107 
    108   ~VCMJitterBuffer();
    109 
    110   // Initializes and starts jitter buffer.
    111   void Start();
    112 
    113   // Signals all internal events and stops the jitter buffer.
    114   void Stop();
    115 
    116   // Returns true if the jitter buffer is running.
    117   bool Running() const;
    118 
    119   // Empty the jitter buffer of all its data.
    120   void Flush();
    121 
    122   // Get the number of received frames, by type, since the jitter buffer
    123   // was started.
    124   FrameCounts FrameStatistics() const;
    125 
    126   // The number of packets discarded by the jitter buffer because the decoder
    127   // won't be able to decode them.
    128   int num_not_decodable_packets() const;
    129 
    130   // Gets number of packets received.
    131   int num_packets() const;
    132 
    133   // Gets number of duplicated packets received.
    134   int num_duplicated_packets() const;
    135 
    136   // Gets number of packets discarded by the jitter buffer.
    137   int num_discarded_packets() const;
    138 
    139   // Statistics, Calculate frame and bit rates.
    140   void IncomingRateStatistics(unsigned int* framerate, unsigned int* bitrate);
    141 
    142   // Checks if the packet sequence will be complete if the next frame would be
    143   // grabbed for decoding. That is, if a frame has been lost between the
    144   // last decoded frame and the next, or if the next frame is missing one
    145   // or more packets.
    146   bool CompleteSequenceWithNextFrame();
    147 
    148   // Wait |max_wait_time_ms| for a complete frame to arrive.
    149   // The function returns true once such a frame is found, its corresponding
    150   // timestamp is returned. Otherwise, returns false.
    151   bool NextCompleteTimestamp(uint32_t max_wait_time_ms, uint32_t* timestamp);
    152 
    153   // Locates a frame for decoding (even an incomplete) without delay.
    154   // The function returns true once such a frame is found, its corresponding
    155   // timestamp is returned. Otherwise, returns false.
    156   bool NextMaybeIncompleteTimestamp(uint32_t* timestamp);
    157 
    158   // Extract frame corresponding to input timestamp.
    159   // Frame will be set to a decoding state.
    160   VCMEncodedFrame* ExtractAndSetDecode(uint32_t timestamp);
    161 
    162   // Releases a frame returned from the jitter buffer, should be called when
    163   // done with decoding.
    164   void ReleaseFrame(VCMEncodedFrame* frame);
    165 
    166   // Returns the time in ms when the latest packet was inserted into the frame.
    167   // Retransmitted is set to true if any of the packets belonging to the frame
    168   // has been retransmitted.
    169   int64_t LastPacketTime(const VCMEncodedFrame* frame,
    170                          bool* retransmitted) const;
    171 
    172   // Inserts a packet into a frame returned from GetFrame().
    173   // If the return value is <= 0, |frame| is invalidated and the pointer must
    174   // be dropped after this function returns.
    175   VCMFrameBufferEnum InsertPacket(const VCMPacket& packet, bool* retransmitted);
    176 
    177   // Returns the estimated jitter in milliseconds.
    178   uint32_t EstimatedJitterMs();
    179 
    180   // Updates the round-trip time estimate.
    181   void UpdateRtt(int64_t rtt_ms);
    182 
    183   // Set the NACK mode. |high_rtt_nack_threshold_ms| is an RTT threshold in ms
    184   // above which NACK will be disabled if the NACK mode is |kNack|, -1 meaning
    185   // that NACK is always enabled in the |kNack| mode.
    186   // |low_rtt_nack_threshold_ms| is an RTT threshold in ms below which we expect
    187   // to rely on NACK only, and therefore are using larger buffers to have time
    188   // to wait for retransmissions.
    189   void SetNackMode(VCMNackMode mode,
    190                    int64_t low_rtt_nack_threshold_ms,
    191                    int64_t high_rtt_nack_threshold_ms);
    192 
    193   void SetNackSettings(size_t max_nack_list_size,
    194                        int max_packet_age_to_nack,
    195                        int max_incomplete_time_ms);
    196 
    197   // Returns the current NACK mode.
    198   VCMNackMode nack_mode() const;
    199 
    200   // Returns a list of the sequence numbers currently missing.
    201   std::vector<uint16_t> GetNackList(bool* request_key_frame);
    202 
    203   // Set decode error mode - Should not be changed in the middle of the
    204   // session. Changes will not influence frames already in the buffer.
    205   void SetDecodeErrorMode(VCMDecodeErrorMode error_mode);
    206   int64_t LastDecodedTimestamp() const;
    207   VCMDecodeErrorMode decode_error_mode() const { return decode_error_mode_; }
    208 
    209   // Used to compute time of complete continuous frames. Returns the timestamps
    210   // corresponding to the start and end of the continuous complete buffer.
    211   void RenderBufferSize(uint32_t* timestamp_start, uint32_t* timestamp_end);
    212 
    213   void RegisterStatsCallback(VCMReceiveStatisticsCallback* callback);
    214 
    215  private:
    216   class SequenceNumberLessThan {
    217    public:
    218     bool operator()(const uint16_t& sequence_number1,
    219                     const uint16_t& sequence_number2) const {
    220       return IsNewerSequenceNumber(sequence_number2, sequence_number1);
    221     }
    222   };
    223   typedef std::set<uint16_t, SequenceNumberLessThan> SequenceNumberSet;
    224 
    225   // Gets the frame assigned to the timestamp of the packet. May recycle
    226   // existing frames if no free frames are available. Returns an error code if
    227   // failing, or kNoError on success. |frame_list| contains which list the
    228   // packet was in, or NULL if it was not in a FrameList (a new frame).
    229   VCMFrameBufferEnum GetFrame(const VCMPacket& packet,
    230                               VCMFrameBuffer** frame,
    231                               FrameList** frame_list)
    232       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    233 
    234   // Returns true if |frame| is continuous in |decoding_state|, not taking
    235   // decodable frames into account.
    236   bool IsContinuousInState(const VCMFrameBuffer& frame,
    237                            const VCMDecodingState& decoding_state) const
    238       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    239   // Returns true if |frame| is continuous in the |last_decoded_state_|, taking
    240   // all decodable frames into account.
    241   bool IsContinuous(const VCMFrameBuffer& frame) const
    242       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    243   // Looks for frames in |incomplete_frames_| which are continuous in the
    244   // provided |decoded_state|. Starts the search from the timestamp of
    245   // |decoded_state|.
    246   void FindAndInsertContinuousFramesWithState(
    247       const VCMDecodingState& decoded_state)
    248       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    249   // Looks for frames in |incomplete_frames_| which are continuous in
    250   // |last_decoded_state_| taking all decodable frames into account. Starts
    251   // the search from |new_frame|.
    252   void FindAndInsertContinuousFrames(const VCMFrameBuffer& new_frame)
    253       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    254   VCMFrameBuffer* NextFrame() const EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    255   // Returns true if the NACK list was updated to cover sequence numbers up to
    256   // |sequence_number|. If false a key frame is needed to get into a state where
    257   // we can continue decoding.
    258   bool UpdateNackList(uint16_t sequence_number)
    259       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    260   bool TooLargeNackList() const;
    261   // Returns true if the NACK list was reduced without problem. If false a key
    262   // frame is needed to get into a state where we can continue decoding.
    263   bool HandleTooLargeNackList() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    264   bool MissingTooOldPacket(uint16_t latest_sequence_number) const
    265       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    266   // Returns true if the too old packets was successfully removed from the NACK
    267   // list. If false, a key frame is needed to get into a state where we can
    268   // continue decoding.
    269   bool HandleTooOldPackets(uint16_t latest_sequence_number)
    270       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    271   // Drops all packets in the NACK list up until |last_decoded_sequence_number|.
    272   void DropPacketsFromNackList(uint16_t last_decoded_sequence_number);
    273 
    274   void ReleaseFrameIfNotDecoding(VCMFrameBuffer* frame);
    275 
    276   // Gets an empty frame, creating a new frame if necessary (i.e. increases
    277   // jitter buffer size).
    278   VCMFrameBuffer* GetEmptyFrame() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    279 
    280   // Attempts to increase the size of the jitter buffer. Returns true on
    281   // success, false otherwise.
    282   bool TryToIncreaseJitterBufferSize() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    283 
    284   // Recycles oldest frames until a key frame is found. Used if jitter buffer is
    285   // completely full. Returns true if a key frame was found.
    286   bool RecycleFramesUntilKeyFrame() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    287 
    288   // Updates the frame statistics.
    289   // Counts only complete frames, so decodable incomplete frames will not be
    290   // counted.
    291   void CountFrame(const VCMFrameBuffer& frame)
    292       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    293 
    294   // Update rolling average of packets per frame.
    295   void UpdateAveragePacketsPerFrame(int current_number_packets_);
    296 
    297   // Cleans the frame list in the JB from old/empty frames.
    298   // Should only be called prior to actual use.
    299   void CleanUpOldOrEmptyFrames() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    300 
    301   // Returns true if |packet| is likely to have been retransmitted.
    302   bool IsPacketRetransmitted(const VCMPacket& packet) const;
    303 
    304   // The following three functions update the jitter estimate with the
    305   // payload size, receive time and RTP timestamp of a frame.
    306   void UpdateJitterEstimate(const VCMJitterSample& sample,
    307                             bool incomplete_frame);
    308   void UpdateJitterEstimate(const VCMFrameBuffer& frame, bool incomplete_frame);
    309   void UpdateJitterEstimate(int64_t latest_packet_time_ms,
    310                             uint32_t timestamp,
    311                             unsigned int frame_size,
    312                             bool incomplete_frame);
    313 
    314   // Returns true if we should wait for retransmissions, false otherwise.
    315   bool WaitForRetransmissions();
    316 
    317   int NonContinuousOrIncompleteDuration() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    318 
    319   uint16_t EstimatedLowSequenceNumber(const VCMFrameBuffer& frame) const;
    320 
    321   void UpdateHistograms() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
    322 
    323   Clock* clock_;
    324   // If we are running (have started) or not.
    325   bool running_;
    326   CriticalSectionWrapper* crit_sect_;
    327   // Event to signal when we have a frame ready for decoder.
    328   rtc::scoped_ptr<EventWrapper> frame_event_;
    329   // Number of allocated frames.
    330   int max_number_of_frames_;
    331   UnorderedFrameList free_frames_ GUARDED_BY(crit_sect_);
    332   FrameList decodable_frames_ GUARDED_BY(crit_sect_);
    333   FrameList incomplete_frames_ GUARDED_BY(crit_sect_);
    334   VCMDecodingState last_decoded_state_ GUARDED_BY(crit_sect_);
    335   bool first_packet_since_reset_;
    336 
    337   // Statistics.
    338   VCMReceiveStatisticsCallback* stats_callback_ GUARDED_BY(crit_sect_);
    339   // Frame counts for each type (key, delta, ...)
    340   FrameCounts receive_statistics_;
    341   // Latest calculated frame rates of incoming stream.
    342   unsigned int incoming_frame_rate_;
    343   unsigned int incoming_frame_count_;
    344   int64_t time_last_incoming_frame_count_;
    345   unsigned int incoming_bit_count_;
    346   unsigned int incoming_bit_rate_;
    347   // Number of frames in a row that have been too old.
    348   int num_consecutive_old_frames_;
    349   // Number of packets in a row that have been too old.
    350   int num_consecutive_old_packets_;
    351   // Number of packets received.
    352   int num_packets_ GUARDED_BY(crit_sect_);
    353   // Number of duplicated packets received.
    354   int num_duplicated_packets_ GUARDED_BY(crit_sect_);
    355   // Number of packets discarded by the jitter buffer.
    356   int num_discarded_packets_ GUARDED_BY(crit_sect_);
    357   // Time when first packet is received.
    358   int64_t time_first_packet_ms_ GUARDED_BY(crit_sect_);
    359 
    360   // Jitter estimation.
    361   // Filter for estimating jitter.
    362   VCMJitterEstimator jitter_estimate_;
    363   // Calculates network delays used for jitter calculations.
    364   VCMInterFrameDelay inter_frame_delay_;
    365   VCMJitterSample waiting_for_completion_;
    366   int64_t rtt_ms_;
    367 
    368   // NACK and retransmissions.
    369   VCMNackMode nack_mode_;
    370   int64_t low_rtt_nack_threshold_ms_;
    371   int64_t high_rtt_nack_threshold_ms_;
    372   // Holds the internal NACK list (the missing sequence numbers).
    373   SequenceNumberSet missing_sequence_numbers_;
    374   uint16_t latest_received_sequence_number_;
    375   size_t max_nack_list_size_;
    376   int max_packet_age_to_nack_;  // Measured in sequence numbers.
    377   int max_incomplete_time_ms_;
    378 
    379   VCMDecodeErrorMode decode_error_mode_;
    380   // Estimated rolling average of packets per frame
    381   float average_packets_per_frame_;
    382   // average_packets_per_frame converges fast if we have fewer than this many
    383   // frames.
    384   int frame_counter_;
    385   RTC_DISALLOW_COPY_AND_ASSIGN(VCMJitterBuffer);
    386 };
    387 }  // namespace webrtc
    388 
    389 #endif  // WEBRTC_MODULES_VIDEO_CODING_JITTER_BUFFER_H_
    390