Home | History | Annotate | Download | only in video_coding 
      1 /*
      2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #ifndef WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
     12 #define WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
     13 
     14 #include <math.h>
     15 #include <stdlib.h>
     16 
     17 #include "webrtc/base/exp_filter.h"
     18 #include "webrtc/base/scoped_ptr.h"
     19 #include "webrtc/modules/video_coding/internal_defines.h"
     20 #include "webrtc/modules/video_coding/qm_select.h"
     21 #include "webrtc/system_wrappers/include/trace.h"
     22 #include "webrtc/typedefs.h"
     23 
     24 namespace webrtc {
     25 namespace media_optimization {
     26 
     27 // Number of time periods used for (max) window filter for packet loss
     28 // TODO(marpan): set reasonable window size for filtered packet loss,
     29 // adjustment should be based on logged/real data of loss stats/correlation.
     30 enum { kLossPrHistorySize = 10 };
     31 
     32 // 1000 ms, total filter length is (kLossPrHistorySize * 1000) ms
     33 enum { kLossPrShortFilterWinMs = 1000 };
     34 
     35 // The type of filter used on the received packet loss reports.
     36 enum FilterPacketLossMode {
     37   kNoFilter,   // No filtering on received loss.
     38   kAvgFilter,  // Recursive average filter.
     39   kMaxFilter   // Max-window filter, over the time interval of:
     40                // (kLossPrHistorySize * kLossPrShortFilterWinMs) ms.
     41 };
     42 
     43 // Thresholds for hybrid NACK/FEC
     44 // common to media optimization and the jitter buffer.
     45 const int64_t kLowRttNackMs = 20;
     46 
     47 struct VCMProtectionParameters {
     48   VCMProtectionParameters()
     49       : rtt(0),
     50         lossPr(0.0f),
     51         bitRate(0.0f),
     52         packetsPerFrame(0.0f),
     53         packetsPerFrameKey(0.0f),
     54         frameRate(0.0f),
     55         keyFrameSize(0.0f),
     56         fecRateDelta(0),
     57         fecRateKey(0),
     58         codecWidth(0),
     59         codecHeight(0),
     60         numLayers(1) {}
     61 
     62   int64_t rtt;
     63   float lossPr;
     64   float bitRate;
     65   float packetsPerFrame;
     66   float packetsPerFrameKey;
     67   float frameRate;
     68   float keyFrameSize;
     69   uint8_t fecRateDelta;
     70   uint8_t fecRateKey;
     71   uint16_t codecWidth;
     72   uint16_t codecHeight;
     73   int numLayers;
     74 };
     75 
     76 /******************************/
     77 /* VCMProtectionMethod class  */
     78 /******************************/
     79 
     80 enum VCMProtectionMethodEnum { kNack, kFec, kNackFec, kNone };
     81 
     82 class VCMLossProbabilitySample {
     83  public:
     84   VCMLossProbabilitySample() : lossPr255(0), timeMs(-1) {}
     85 
     86   uint8_t lossPr255;
     87   int64_t timeMs;
     88 };
     89 
     90 class VCMProtectionMethod {
     91  public:
     92   VCMProtectionMethod();
     93   virtual ~VCMProtectionMethod();
     94 
     95   // Updates the efficiency of the method using the parameters provided
     96   //
     97   // Input:
     98   //         - parameters         : Parameters used to calculate efficiency
     99   //
    100   // Return value                 : True if this method is recommended in
    101   //                                the given conditions.
    102   virtual bool UpdateParameters(const VCMProtectionParameters* parameters) = 0;
    103 
    104   // Returns the protection type
    105   //
    106   // Return value                 : The protection type
    107   enum VCMProtectionMethodEnum Type() const { return _type; }
    108 
    109   // Returns the effective packet loss for ER, required by this protection
    110   // method
    111   //
    112   // Return value                 : Required effective packet loss
    113   virtual uint8_t RequiredPacketLossER() { return _effectivePacketLoss; }
    114 
    115   // Extracts the FEC protection factor for Key frame, required by this
    116   // protection method
    117   //
    118   // Return value                 : Required protectionFactor for Key frame
    119   virtual uint8_t RequiredProtectionFactorK() { return _protectionFactorK; }
    120 
    121   // Extracts the FEC protection factor for Delta frame, required by this
    122   // protection method
    123   //
    124   // Return value                 : Required protectionFactor for delta frame
    125   virtual uint8_t RequiredProtectionFactorD() { return _protectionFactorD; }
    126 
    127   // Extracts whether the FEC Unequal protection (UEP) is used for Key frame.
    128   //
    129   // Return value                 : Required Unequal protection on/off state.
    130   virtual bool RequiredUepProtectionK() { return _useUepProtectionK; }
    131 
    132   // Extracts whether the the FEC Unequal protection (UEP) is used for Delta
    133   // frame.
    134   //
    135   // Return value                 : Required Unequal protection on/off state.
    136   virtual bool RequiredUepProtectionD() { return _useUepProtectionD; }
    137 
    138   virtual int MaxFramesFec() const { return 1; }
    139 
    140   // Updates content metrics
    141   void UpdateContentMetrics(const VideoContentMetrics* contentMetrics);
    142 
    143  protected:
    144   uint8_t _effectivePacketLoss;
    145   uint8_t _protectionFactorK;
    146   uint8_t _protectionFactorD;
    147   // Estimation of residual loss after the FEC
    148   float _scaleProtKey;
    149   int32_t _maxPayloadSize;
    150 
    151   VCMQmRobustness* _qmRobustness;
    152   bool _useUepProtectionK;
    153   bool _useUepProtectionD;
    154   float _corrFecCost;
    155   enum VCMProtectionMethodEnum _type;
    156 };
    157 
    158 class VCMNackMethod : public VCMProtectionMethod {
    159  public:
    160   VCMNackMethod();
    161   virtual ~VCMNackMethod();
    162   virtual bool UpdateParameters(const VCMProtectionParameters* parameters);
    163   // Get the effective packet loss
    164   bool EffectivePacketLoss(const VCMProtectionParameters* parameter);
    165 };
    166 
    167 class VCMFecMethod : public VCMProtectionMethod {
    168  public:
    169   VCMFecMethod();
    170   virtual ~VCMFecMethod();
    171   virtual bool UpdateParameters(const VCMProtectionParameters* parameters);
    172   // Get the effective packet loss for ER
    173   bool EffectivePacketLoss(const VCMProtectionParameters* parameters);
    174   // Get the FEC protection factors
    175   bool ProtectionFactor(const VCMProtectionParameters* parameters);
    176   // Get the boost for key frame protection
    177   uint8_t BoostCodeRateKey(uint8_t packetFrameDelta,
    178                            uint8_t packetFrameKey) const;
    179   // Convert the rates: defined relative to total# packets or source# packets
    180   uint8_t ConvertFECRate(uint8_t codeRate) const;
    181   // Get the average effective recovery from FEC: for random loss model
    182   float AvgRecoveryFEC(const VCMProtectionParameters* parameters) const;
    183   // Update FEC with protectionFactorD
    184   void UpdateProtectionFactorD(uint8_t protectionFactorD);
    185   // Update FEC with protectionFactorK
    186   void UpdateProtectionFactorK(uint8_t protectionFactorK);
    187   // Compute the bits per frame. Account for temporal layers when applicable.
    188   int BitsPerFrame(const VCMProtectionParameters* parameters);
    189 
    190  protected:
    191   enum { kUpperLimitFramesFec = 6 };
    192   // Thresholds values for the bytes/frame and round trip time, below which we
    193   // may turn off FEC, depending on |_numLayers| and |_maxFramesFec|.
    194   // Max bytes/frame for VGA, corresponds to ~140k at 25fps.
    195   enum { kMaxBytesPerFrameForFec = 700 };
    196   // Max bytes/frame for CIF and lower: corresponds to ~80k at 25fps.
    197   enum { kMaxBytesPerFrameForFecLow = 400 };
    198   // Max bytes/frame for frame size larger than VGA, ~200k at 25fps.
    199   enum { kMaxBytesPerFrameForFecHigh = 1000 };
    200 };
    201 
    202 class VCMNackFecMethod : public VCMFecMethod {
    203  public:
    204   VCMNackFecMethod(int64_t lowRttNackThresholdMs,
    205                    int64_t highRttNackThresholdMs);
    206   virtual ~VCMNackFecMethod();
    207   virtual bool UpdateParameters(const VCMProtectionParameters* parameters);
    208   // Get the effective packet loss for ER
    209   bool EffectivePacketLoss(const VCMProtectionParameters* parameters);
    210   // Get the protection factors
    211   bool ProtectionFactor(const VCMProtectionParameters* parameters);
    212   // Get the max number of frames the FEC is allowed to be based on.
    213   int MaxFramesFec() const;
    214   // Turn off the FEC based on low bitrate and other factors.
    215   bool BitRateTooLowForFec(const VCMProtectionParameters* parameters);
    216 
    217  private:
    218   int ComputeMaxFramesFec(const VCMProtectionParameters* parameters);
    219 
    220   int64_t _lowRttNackMs;
    221   int64_t _highRttNackMs;
    222   int _maxFramesFec;
    223 };
    224 
    225 class VCMLossProtectionLogic {
    226  public:
    227   explicit VCMLossProtectionLogic(int64_t nowMs);
    228   ~VCMLossProtectionLogic();
    229 
    230   // Set the protection method to be used
    231   //
    232   // Input:
    233   //        - newMethodType    : New requested protection method type. If one
    234   //                           is already set, it will be deleted and replaced
    235   void SetMethod(VCMProtectionMethodEnum newMethodType);
    236 
    237   // Update the round-trip time
    238   //
    239   // Input:
    240   //          - rtt           : Round-trip time in seconds.
    241   void UpdateRtt(int64_t rtt);
    242 
    243   // Update the filtered packet loss.
    244   //
    245   // Input:
    246   //          - packetLossEnc :  The reported packet loss filtered
    247   //                             (max window or average)
    248   void UpdateFilteredLossPr(uint8_t packetLossEnc);
    249 
    250   // Update the current target bit rate.
    251   //
    252   // Input:
    253   //          - bitRate          : The current target bit rate in kbits/s
    254   void UpdateBitRate(float bitRate);
    255 
    256   // Update the number of packets per frame estimate, for delta frames
    257   //
    258   // Input:
    259   //          - nPackets         : Number of packets in the latest sent frame.
    260   void UpdatePacketsPerFrame(float nPackets, int64_t nowMs);
    261 
    262   // Update the number of packets per frame estimate, for key frames
    263   //
    264   // Input:
    265   //          - nPackets         : umber of packets in the latest sent frame.
    266   void UpdatePacketsPerFrameKey(float nPackets, int64_t nowMs);
    267 
    268   // Update the keyFrameSize estimate
    269   //
    270   // Input:
    271   //          - keyFrameSize     : The size of the latest sent key frame.
    272   void UpdateKeyFrameSize(float keyFrameSize);
    273 
    274   // Update the frame rate
    275   //
    276   // Input:
    277   //          - frameRate        : The current target frame rate.
    278   void UpdateFrameRate(float frameRate) { _frameRate = frameRate; }
    279 
    280   // Update the frame size
    281   //
    282   // Input:
    283   //          - width        : The codec frame width.
    284   //          - height       : The codec frame height.
    285   void UpdateFrameSize(uint16_t width, uint16_t height);
    286 
    287   // Update the number of active layers
    288   //
    289   // Input:
    290   //          - numLayers    : Number of layers used.
    291   void UpdateNumLayers(int numLayers);
    292 
    293   // The amount of packet loss to cover for with FEC.
    294   //
    295   // Input:
    296   //          - fecRateKey      : Packet loss to cover for with FEC when
    297   //                              sending key frames.
    298   //          - fecRateDelta    : Packet loss to cover for with FEC when
    299   //                              sending delta frames.
    300   void UpdateFECRates(uint8_t fecRateKey, uint8_t fecRateDelta) {
    301     _fecRateKey = fecRateKey;
    302     _fecRateDelta = fecRateDelta;
    303   }
    304 
    305   // Update the protection methods with the current VCMProtectionParameters
    306   // and set the requested protection settings.
    307   // Return value     : Returns true on update
    308   bool UpdateMethod();
    309 
    310   // Returns the method currently selected.
    311   //
    312   // Return value                 : The protection method currently selected.
    313   VCMProtectionMethod* SelectedMethod() const;
    314 
    315   // Return the protection type of the currently selected method
    316   VCMProtectionMethodEnum SelectedType() const;
    317 
    318   // Updates the filtered loss for the average and max window packet loss,
    319   // and returns the filtered loss probability in the interval [0, 255].
    320   // The returned filtered loss value depends on the parameter |filter_mode|.
    321   // The input parameter |lossPr255| is the received packet loss.
    322 
    323   // Return value                 : The filtered loss probability
    324   uint8_t FilteredLoss(int64_t nowMs,
    325                        FilterPacketLossMode filter_mode,
    326                        uint8_t lossPr255);
    327 
    328   void Reset(int64_t nowMs);
    329 
    330   void Release();
    331 
    332  private:
    333   // Sets the available loss protection methods.
    334   void UpdateMaxLossHistory(uint8_t lossPr255, int64_t now);
    335   uint8_t MaxFilteredLossPr(int64_t nowMs) const;
    336   rtc::scoped_ptr<VCMProtectionMethod> _selectedMethod;
    337   VCMProtectionParameters _currentParameters;
    338   int64_t _rtt;
    339   float _lossPr;
    340   float _bitRate;
    341   float _frameRate;
    342   float _keyFrameSize;
    343   uint8_t _fecRateKey;
    344   uint8_t _fecRateDelta;
    345   int64_t _lastPrUpdateT;
    346   int64_t _lastPacketPerFrameUpdateT;
    347   int64_t _lastPacketPerFrameUpdateTKey;
    348   rtc::ExpFilter _lossPr255;
    349   VCMLossProbabilitySample _lossPrHistory[kLossPrHistorySize];
    350   uint8_t _shortMaxLossPr255;
    351   rtc::ExpFilter _packetsPerFrame;
    352   rtc::ExpFilter _packetsPerFrameKey;
    353   uint16_t _codecWidth;
    354   uint16_t _codecHeight;
    355   int _numLayers;
    356 };
    357 
    358 }  // namespace media_optimization
    359 }  // namespace webrtc
    360 
    361 #endif  // WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
    362