Home | History | Annotate | Download | only in vp8 
      1 /* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
      2 *
      3 *  Use of this source code is governed by a BSD-style license
      4 *  that can be found in the LICENSE file in the root of the source
      5 *  tree. An additional intellectual property rights grant can be found
      6 *  in the file PATENTS.  All contributing project authors may
      7 *  be found in the AUTHORS file in the root of the source tree.
      8 */
      9 
     10 #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
     11 
     12 #include <stdlib.h>
     13 
     14 #include <algorithm>
     15 
     16 #include "webrtc/base/checks.h"
     17 #include "vpx/vpx_encoder.h"
     18 #include "vpx/vp8cx.h"
     19 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
     20 
     21 namespace webrtc {
     22 
     23 static const int kOneSecond90Khz = 90000;
     24 static const int kMinTimeBetweenSyncs = kOneSecond90Khz * 5;
     25 static const int kMaxTimeBetweenSyncs = kOneSecond90Khz * 10;
     26 static const int kQpDeltaThresholdForSync = 8;
     27 
     28 const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
     29 const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
     30 
     31 // Since this is TL0 we only allow updating and predicting from the LAST
     32 // reference frame.
     33 const int ScreenshareLayers::kTl0Flags =
     34     VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF |
     35     VP8_EFLAG_NO_REF_ARF;
     36 
     37 // Allow predicting from both TL0 and TL1.
     38 const int ScreenshareLayers::kTl1Flags =
     39     VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
     40 
     41 // Allow predicting from only TL0 to allow participants to switch to the high
     42 // bitrate stream. This means predicting only from the LAST reference frame, but
     43 // only updating GF to not corrupt TL0.
     44 const int ScreenshareLayers::kTl1SyncFlags =
     45     VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF |
     46     VP8_EFLAG_NO_UPD_LAST;
     47 
     48 ScreenshareLayers::ScreenshareLayers(int num_temporal_layers,
     49                                      uint8_t initial_tl0_pic_idx)
     50     : number_of_temporal_layers_(num_temporal_layers),
     51       last_base_layer_sync_(false),
     52       tl0_pic_idx_(initial_tl0_pic_idx),
     53       active_layer_(-1),
     54       last_timestamp_(-1),
     55       last_sync_timestamp_(-1),
     56       min_qp_(-1),
     57       max_qp_(-1),
     58       max_debt_bytes_(0),
     59       frame_rate_(-1) {
     60   assert(num_temporal_layers > 0);
     61   assert(num_temporal_layers <= 2);
     62 }
     63 
     64 int ScreenshareLayers::CurrentLayerId() const {
     65   // Codec does not use temporal layers for screenshare.
     66   return 0;
     67 }
     68 
     69 int ScreenshareLayers::EncodeFlags(uint32_t timestamp) {
     70   if (number_of_temporal_layers_ <= 1) {
     71     // No flags needed for 1 layer screenshare.
     72     return 0;
     73   }
     74 
     75   int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
     76   int flags = 0;
     77 
     78   if (active_layer_ == -1 ||
     79       layers_[active_layer_].state != TemporalLayer::State::kDropped) {
     80     if (layers_[0].debt_bytes_ > max_debt_bytes_) {
     81       // Must drop TL0, encode TL1 instead.
     82       if (layers_[1].debt_bytes_ > max_debt_bytes_) {
     83         // Must drop both TL0 and TL1.
     84         active_layer_ = -1;
     85       } else {
     86         active_layer_ = 1;
     87       }
     88     } else {
     89       active_layer_ = 0;
     90     }
     91   }
     92 
     93   switch (active_layer_) {
     94     case 0:
     95       flags = kTl0Flags;
     96       break;
     97     case 1:
     98       if (TimeToSync(unwrapped_timestamp)) {
     99         last_sync_timestamp_ = unwrapped_timestamp;
    100         flags = kTl1SyncFlags;
    101       } else {
    102         flags = kTl1Flags;
    103       }
    104       break;
    105     case -1:
    106       flags = -1;
    107       break;
    108     default:
    109       flags = -1;
    110       RTC_NOTREACHED();
    111   }
    112 
    113   // Make sure both frame droppers leak out bits.
    114   int64_t ts_diff;
    115   if (last_timestamp_ == -1) {
    116     ts_diff = kOneSecond90Khz / (frame_rate_ <= 0 ? 5 : frame_rate_);
    117   } else {
    118     ts_diff = unwrapped_timestamp - last_timestamp_;
    119   }
    120 
    121   layers_[0].UpdateDebt(ts_diff / 90);
    122   layers_[1].UpdateDebt(ts_diff / 90);
    123   last_timestamp_ = timestamp;
    124   return flags;
    125 }
    126 
    127 bool ScreenshareLayers::ConfigureBitrates(int bitrate_kbps,
    128                                           int max_bitrate_kbps,
    129                                           int framerate,
    130                                           vpx_codec_enc_cfg_t* cfg) {
    131   layers_[0].target_rate_kbps_ = bitrate_kbps;
    132   layers_[1].target_rate_kbps_ = max_bitrate_kbps;
    133 
    134   int target_bitrate_kbps = bitrate_kbps;
    135 
    136   if (cfg != nullptr) {
    137     if (number_of_temporal_layers_ > 1) {
    138       // Calculate a codec target bitrate. This may be higher than TL0, gaining
    139       // quality at the expense of frame rate at TL0. Constraints:
    140       // - TL0 frame rate no less than framerate / kMaxTL0FpsReduction.
    141       // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate.
    142       target_bitrate_kbps =
    143           std::min(bitrate_kbps * kMaxTL0FpsReduction,
    144                    max_bitrate_kbps / kAcceptableTargetOvershoot);
    145 
    146       cfg->rc_target_bitrate = std::max(bitrate_kbps, target_bitrate_kbps);
    147     }
    148 
    149     // Don't reconfigure qp limits during quality boost frames.
    150     if (layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) {
    151       min_qp_ = cfg->rc_min_quantizer;
    152       max_qp_ = cfg->rc_max_quantizer;
    153       // After a dropped frame, a frame with max qp will be encoded and the
    154       // quality will then ramp up from there. To boost the speed of recovery,
    155       // encode the next frame with lower max qp. TL0 is the most important to
    156       // improve since the errors in this layer will propagate to TL1.
    157       // Currently, reduce max qp by 20% for TL0 and 15% for TL1.
    158       layers_[0].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 80) / 100);
    159       layers_[1].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 85) / 100);
    160     }
    161   }
    162 
    163   int avg_frame_size = (target_bitrate_kbps * 1000) / (8 * framerate);
    164   max_debt_bytes_ = 4 * avg_frame_size;
    165 
    166   return true;
    167 }
    168 
    169 void ScreenshareLayers::FrameEncoded(unsigned int size,
    170                                      uint32_t timestamp,
    171                                      int qp) {
    172   if (size == 0) {
    173     layers_[active_layer_].state = TemporalLayer::State::kDropped;
    174     return;
    175   }
    176 
    177   if (layers_[active_layer_].state == TemporalLayer::State::kDropped) {
    178     layers_[active_layer_].state = TemporalLayer::State::kQualityBoost;
    179   }
    180 
    181   if (qp != -1)
    182     layers_[active_layer_].last_qp = qp;
    183 
    184   if (active_layer_ == 0) {
    185     layers_[0].debt_bytes_ += size;
    186     layers_[1].debt_bytes_ += size;
    187   } else if (active_layer_ == 1) {
    188     layers_[1].debt_bytes_ += size;
    189   }
    190 }
    191 
    192 void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync,
    193                                               CodecSpecificInfoVP8* vp8_info,
    194                                               uint32_t timestamp) {
    195   int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
    196   if (number_of_temporal_layers_ == 1) {
    197     vp8_info->temporalIdx = kNoTemporalIdx;
    198     vp8_info->layerSync = false;
    199     vp8_info->tl0PicIdx = kNoTl0PicIdx;
    200   } else {
    201     vp8_info->temporalIdx = active_layer_;
    202     if (base_layer_sync) {
    203       vp8_info->temporalIdx = 0;
    204       last_sync_timestamp_ = unwrapped_timestamp;
    205     } else if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) {
    206       // Regardless of pattern the frame after a base layer sync will always
    207       // be a layer sync.
    208       last_sync_timestamp_ = unwrapped_timestamp;
    209     }
    210     vp8_info->layerSync = last_sync_timestamp_ != -1 &&
    211                           last_sync_timestamp_ == unwrapped_timestamp;
    212     if (vp8_info->temporalIdx == 0) {
    213       tl0_pic_idx_++;
    214     }
    215     last_base_layer_sync_ = base_layer_sync;
    216     vp8_info->tl0PicIdx = tl0_pic_idx_;
    217   }
    218 }
    219 
    220 bool ScreenshareLayers::TimeToSync(int64_t timestamp) const {
    221   if (active_layer_ != 1) {
    222     RTC_NOTREACHED();
    223     return false;
    224   }
    225   RTC_DCHECK_NE(-1, layers_[0].last_qp);
    226   if (layers_[1].last_qp == -1) {
    227     // First frame in TL1 should only depend on TL0 since there are no
    228     // previous frames in TL1.
    229     return true;
    230   }
    231 
    232   RTC_DCHECK_NE(-1, last_sync_timestamp_);
    233   int64_t timestamp_diff = timestamp - last_sync_timestamp_;
    234   if (timestamp_diff > kMaxTimeBetweenSyncs) {
    235     // After a certain time, force a sync frame.
    236     return true;
    237   } else if (timestamp_diff < kMinTimeBetweenSyncs) {
    238     // If too soon from previous sync frame, don't issue a new one.
    239     return false;
    240   }
    241   // Issue a sync frame if difference in quality between TL0 and TL1 isn't too
    242   // large.
    243   if (layers_[0].last_qp - layers_[1].last_qp < kQpDeltaThresholdForSync)
    244     return true;
    245   return false;
    246 }
    247 
    248 bool ScreenshareLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) {
    249   if (max_qp_ == -1 || number_of_temporal_layers_ <= 1)
    250     return false;
    251 
    252   // If layer is in the quality boost state (following a dropped frame), update
    253   // the configuration with the adjusted (lower) qp and set the state back to
    254   // normal.
    255   unsigned int adjusted_max_qp;
    256   if (layers_[active_layer_].state == TemporalLayer::State::kQualityBoost &&
    257       layers_[active_layer_].enhanced_max_qp != -1) {
    258     adjusted_max_qp = layers_[active_layer_].enhanced_max_qp;
    259     layers_[active_layer_].state = TemporalLayer::State::kNormal;
    260   } else {
    261     if (max_qp_ == -1)
    262       return false;
    263     adjusted_max_qp = max_qp_;  // Set the normal max qp.
    264   }
    265 
    266   if (adjusted_max_qp == cfg->rc_max_quantizer)
    267     return false;
    268 
    269   cfg->rc_max_quantizer = adjusted_max_qp;
    270   return true;
    271 }
    272 
    273 void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) {
    274   uint32_t debt_reduction_bytes = target_rate_kbps_ * delta_ms / 8;
    275   if (debt_reduction_bytes >= debt_bytes_) {
    276     debt_bytes_ = 0;
    277   } else {
    278     debt_bytes_ -= debt_reduction_bytes;
    279   }
    280 }
    281 
    282 }  // namespace webrtc
    283