Home | History | Annotate | Download | only in source
      1 /*
      2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/modules/video_coding/main/source/media_optimization.h"
     12 
     13 #include "webrtc/modules/video_coding/main/source/content_metrics_processing.h"
     14 #include "webrtc/modules/video_coding/main/source/qm_select.h"
     15 #include "webrtc/modules/video_coding/utility/include/frame_dropper.h"
     16 #include "webrtc/system_wrappers/interface/clock.h"
     17 #include "webrtc/system_wrappers/interface/logging.h"
     18 
     19 namespace webrtc {
     20 namespace media_optimization {
     21 namespace {
     22 void UpdateProtectionCallback(
     23     VCMProtectionMethod* selected_method,
     24     uint32_t* video_rate_bps,
     25     uint32_t* nack_overhead_rate_bps,
     26     uint32_t* fec_overhead_rate_bps,
     27     VCMProtectionCallback* video_protection_callback) {
     28   FecProtectionParams delta_fec_params;
     29   FecProtectionParams key_fec_params;
     30   // Get the FEC code rate for Key frames (set to 0 when NA).
     31   key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK();
     32 
     33   // Get the FEC code rate for Delta frames (set to 0 when NA).
     34   delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD();
     35 
     36   // Get the FEC-UEP protection status for Key frames: UEP on/off.
     37   key_fec_params.use_uep_protection = selected_method->RequiredUepProtectionK();
     38 
     39   // Get the FEC-UEP protection status for Delta frames: UEP on/off.
     40   delta_fec_params.use_uep_protection =
     41       selected_method->RequiredUepProtectionD();
     42 
     43   // The RTP module currently requires the same |max_fec_frames| for both
     44   // key and delta frames.
     45   delta_fec_params.max_fec_frames = selected_method->MaxFramesFec();
     46   key_fec_params.max_fec_frames = selected_method->MaxFramesFec();
     47 
     48   // Set the FEC packet mask type. |kFecMaskBursty| is more effective for
     49   // consecutive losses and little/no packet re-ordering. As we currently
     50   // do not have feedback data on the degree of correlated losses and packet
     51   // re-ordering, we keep default setting to |kFecMaskRandom| for now.
     52   delta_fec_params.fec_mask_type = kFecMaskRandom;
     53   key_fec_params.fec_mask_type = kFecMaskRandom;
     54 
     55   // TODO(Marco): Pass FEC protection values per layer.
     56   video_protection_callback->ProtectionRequest(&delta_fec_params,
     57                                                &key_fec_params,
     58                                                video_rate_bps,
     59                                                nack_overhead_rate_bps,
     60                                                fec_overhead_rate_bps);
     61 }
     62 }  // namespace
     63 
     64 struct MediaOptimization::EncodedFrameSample {
     65   EncodedFrameSample(int size_bytes,
     66                      uint32_t timestamp,
     67                      int64_t time_complete_ms)
     68       : size_bytes(size_bytes),
     69         timestamp(timestamp),
     70         time_complete_ms(time_complete_ms) {}
     71 
     72   uint32_t size_bytes;
     73   uint32_t timestamp;
     74   int64_t time_complete_ms;
     75 };
     76 
     77 MediaOptimization::MediaOptimization(Clock* clock)
     78     : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
     79       clock_(clock),
     80       max_bit_rate_(0),
     81       send_codec_type_(kVideoCodecUnknown),
     82       codec_width_(0),
     83       codec_height_(0),
     84       user_frame_rate_(0),
     85       frame_dropper_(new FrameDropper),
     86       loss_prot_logic_(
     87           new VCMLossProtectionLogic(clock_->TimeInMilliseconds())),
     88       fraction_lost_(0),
     89       send_statistics_zero_encode_(0),
     90       max_payload_size_(1460),
     91       target_bit_rate_(0),
     92       incoming_frame_rate_(0),
     93       enable_qm_(false),
     94       encoded_frame_samples_(),
     95       avg_sent_bit_rate_bps_(0),
     96       avg_sent_framerate_(0),
     97       key_frame_cnt_(0),
     98       delta_frame_cnt_(0),
     99       content_(new VCMContentMetricsProcessing()),
    100       qm_resolution_(new VCMQmResolution()),
    101       last_qm_update_time_(0),
    102       last_change_time_(0),
    103       num_layers_(0),
    104       suspension_enabled_(false),
    105       video_suspended_(false),
    106       suspension_threshold_bps_(0),
    107       suspension_window_bps_(0) {
    108   memset(send_statistics_, 0, sizeof(send_statistics_));
    109   memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
    110 }
    111 
    112 MediaOptimization::~MediaOptimization(void) {
    113   loss_prot_logic_->Release();
    114 }
    115 
    116 void MediaOptimization::Reset() {
    117   CriticalSectionScoped lock(crit_sect_.get());
    118   SetEncodingDataInternal(
    119       kVideoCodecUnknown, 0, 0, 0, 0, 0, 0, max_payload_size_);
    120   memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
    121   incoming_frame_rate_ = 0.0;
    122   frame_dropper_->Reset();
    123   loss_prot_logic_->Reset(clock_->TimeInMilliseconds());
    124   frame_dropper_->SetRates(0, 0);
    125   content_->Reset();
    126   qm_resolution_->Reset();
    127   loss_prot_logic_->UpdateFrameRate(incoming_frame_rate_);
    128   loss_prot_logic_->Reset(clock_->TimeInMilliseconds());
    129   send_statistics_zero_encode_ = 0;
    130   target_bit_rate_ = 0;
    131   codec_width_ = 0;
    132   codec_height_ = 0;
    133   user_frame_rate_ = 0;
    134   key_frame_cnt_ = 0;
    135   delta_frame_cnt_ = 0;
    136   last_qm_update_time_ = 0;
    137   last_change_time_ = 0;
    138   encoded_frame_samples_.clear();
    139   avg_sent_bit_rate_bps_ = 0;
    140   num_layers_ = 1;
    141 }
    142 
    143 void MediaOptimization::SetEncodingData(VideoCodecType send_codec_type,
    144                                         int32_t max_bit_rate,
    145                                         uint32_t frame_rate,
    146                                         uint32_t target_bitrate,
    147                                         uint16_t width,
    148                                         uint16_t height,
    149                                         int num_layers,
    150                                         int32_t mtu) {
    151   CriticalSectionScoped lock(crit_sect_.get());
    152   SetEncodingDataInternal(send_codec_type,
    153                           max_bit_rate,
    154                           frame_rate,
    155                           target_bitrate,
    156                           width,
    157                           height,
    158                           num_layers,
    159                           mtu);
    160 }
    161 
    162 void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type,
    163                                                 int32_t max_bit_rate,
    164                                                 uint32_t frame_rate,
    165                                                 uint32_t target_bitrate,
    166                                                 uint16_t width,
    167                                                 uint16_t height,
    168                                                 int num_layers,
    169                                                 int32_t mtu) {
    170   // Everything codec specific should be reset here since this means the codec
    171   // has changed. If native dimension values have changed, then either user
    172   // initiated change, or QM initiated change. Will be able to determine only
    173   // after the processing of the first frame.
    174   last_change_time_ = clock_->TimeInMilliseconds();
    175   content_->Reset();
    176   content_->UpdateFrameRate(frame_rate);
    177 
    178   max_bit_rate_ = max_bit_rate;
    179   send_codec_type_ = send_codec_type;
    180   target_bit_rate_ = target_bitrate;
    181   float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
    182   loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
    183   loss_prot_logic_->UpdateFrameRate(static_cast<float>(frame_rate));
    184   loss_prot_logic_->UpdateFrameSize(width, height);
    185   loss_prot_logic_->UpdateNumLayers(num_layers);
    186   frame_dropper_->Reset();
    187   frame_dropper_->SetRates(target_bitrate_kbps, static_cast<float>(frame_rate));
    188   user_frame_rate_ = static_cast<float>(frame_rate);
    189   codec_width_ = width;
    190   codec_height_ = height;
    191   num_layers_ = (num_layers <= 1) ? 1 : num_layers;  // Can also be zero.
    192   max_payload_size_ = mtu;
    193   qm_resolution_->Initialize(target_bitrate_kbps,
    194                              user_frame_rate_,
    195                              codec_width_,
    196                              codec_height_,
    197                              num_layers_);
    198 }
    199 
    200 uint32_t MediaOptimization::SetTargetRates(
    201     uint32_t target_bitrate,
    202     uint8_t fraction_lost,
    203     uint32_t round_trip_time_ms,
    204     VCMProtectionCallback* protection_callback,
    205     VCMQMSettingsCallback* qmsettings_callback) {
    206   CriticalSectionScoped lock(crit_sect_.get());
    207   // TODO(holmer): Consider putting this threshold only on the video bitrate,
    208   // and not on protection.
    209   if (max_bit_rate_ > 0 &&
    210       target_bitrate > static_cast<uint32_t>(max_bit_rate_)) {
    211     target_bitrate = max_bit_rate_;
    212   }
    213   VCMProtectionMethod* selected_method = loss_prot_logic_->SelectedMethod();
    214   float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
    215   loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
    216   loss_prot_logic_->UpdateRtt(round_trip_time_ms);
    217   loss_prot_logic_->UpdateResidualPacketLoss(static_cast<float>(fraction_lost));
    218 
    219   // Get frame rate for encoder: this is the actual/sent frame rate.
    220   float actual_frame_rate = SentFrameRateInternal();
    221 
    222   // Sanity check.
    223   if (actual_frame_rate < 1.0) {
    224     actual_frame_rate = 1.0;
    225   }
    226 
    227   // Update frame rate for the loss protection logic class: frame rate should
    228   // be the actual/sent rate.
    229   loss_prot_logic_->UpdateFrameRate(actual_frame_rate);
    230 
    231   fraction_lost_ = fraction_lost;
    232 
    233   // Returns the filtered packet loss, used for the protection setting.
    234   // The filtered loss may be the received loss (no filter), or some
    235   // filtered value (average or max window filter).
    236   // Use max window filter for now.
    237   FilterPacketLossMode filter_mode = kMaxFilter;
    238   uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss(
    239       clock_->TimeInMilliseconds(), filter_mode, fraction_lost);
    240 
    241   // For now use the filtered loss for computing the robustness settings.
    242   loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc);
    243 
    244   // Rate cost of the protection methods.
    245   uint32_t protection_overhead_bps = 0;
    246 
    247   // Update protection settings, when applicable.
    248   float sent_video_rate_kbps = 0.0f;
    249   if (selected_method) {
    250     // Update protection method with content metrics.
    251     selected_method->UpdateContentMetrics(content_->ShortTermAvgData());
    252 
    253     // Update method will compute the robustness settings for the given
    254     // protection method and the overhead cost
    255     // the protection method is set by the user via SetVideoProtection.
    256     loss_prot_logic_->UpdateMethod();
    257 
    258     // Update protection callback with protection settings.
    259     uint32_t sent_video_rate_bps = 0;
    260     uint32_t sent_nack_rate_bps = 0;
    261     uint32_t sent_fec_rate_bps = 0;
    262     // Get the bit cost of protection method, based on the amount of
    263     // overhead data actually transmitted (including headers) the last
    264     // second.
    265     if (protection_callback) {
    266       UpdateProtectionCallback(selected_method,
    267                                &sent_video_rate_bps,
    268                                &sent_nack_rate_bps,
    269                                &sent_fec_rate_bps,
    270                                protection_callback);
    271     }
    272     uint32_t sent_total_rate_bps =
    273         sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps;
    274     // Estimate the overhead costs of the next second as staying the same
    275     // wrt the source bitrate.
    276     if (sent_total_rate_bps > 0) {
    277       protection_overhead_bps = static_cast<uint32_t>(
    278           target_bitrate *
    279               static_cast<double>(sent_nack_rate_bps + sent_fec_rate_bps) /
    280               sent_total_rate_bps +
    281           0.5);
    282     }
    283     // Cap the overhead estimate to 50%.
    284     if (protection_overhead_bps > target_bitrate / 2)
    285       protection_overhead_bps = target_bitrate / 2;
    286 
    287     // Get the effective packet loss for encoder ER when applicable. Should be
    288     // passed to encoder via fraction_lost.
    289     packet_loss_enc = selected_method->RequiredPacketLossER();
    290     sent_video_rate_kbps = static_cast<float>(sent_video_rate_bps) / 1000.0f;
    291   }
    292 
    293   // Source coding rate: total rate - protection overhead.
    294   target_bit_rate_ = target_bitrate - protection_overhead_bps;
    295 
    296   // Update encoding rates following protection settings.
    297   float target_video_bitrate_kbps =
    298       static_cast<float>(target_bit_rate_) / 1000.0f;
    299   frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_);
    300 
    301   if (enable_qm_ && qmsettings_callback) {
    302     // Update QM with rates.
    303     qm_resolution_->UpdateRates(target_video_bitrate_kbps,
    304                                 sent_video_rate_kbps,
    305                                 incoming_frame_rate_,
    306                                 fraction_lost_);
    307     // Check for QM selection.
    308     bool select_qm = CheckStatusForQMchange();
    309     if (select_qm) {
    310       SelectQuality(qmsettings_callback);
    311     }
    312     // Reset the short-term averaged content data.
    313     content_->ResetShortTermAvgData();
    314   }
    315 
    316   CheckSuspendConditions();
    317 
    318   return target_bit_rate_;
    319 }
    320 
    321 void MediaOptimization::EnableProtectionMethod(bool enable,
    322                                                VCMProtectionMethodEnum method) {
    323   CriticalSectionScoped lock(crit_sect_.get());
    324   bool updated = false;
    325   if (enable) {
    326     updated = loss_prot_logic_->SetMethod(method);
    327   } else {
    328     loss_prot_logic_->RemoveMethod(method);
    329   }
    330   if (updated) {
    331     loss_prot_logic_->UpdateMethod();
    332   }
    333 }
    334 
    335 uint32_t MediaOptimization::InputFrameRate() {
    336   CriticalSectionScoped lock(crit_sect_.get());
    337   return InputFrameRateInternal();
    338 }
    339 
    340 uint32_t MediaOptimization::InputFrameRateInternal() {
    341   ProcessIncomingFrameRate(clock_->TimeInMilliseconds());
    342   return uint32_t(incoming_frame_rate_ + 0.5f);
    343 }
    344 
    345 uint32_t MediaOptimization::SentFrameRate() {
    346   CriticalSectionScoped lock(crit_sect_.get());
    347   return SentFrameRateInternal();
    348 }
    349 
    350 uint32_t MediaOptimization::SentFrameRateInternal() {
    351   PurgeOldFrameSamples(clock_->TimeInMilliseconds());
    352   UpdateSentFramerate();
    353   return avg_sent_framerate_;
    354 }
    355 
    356 uint32_t MediaOptimization::SentBitRate() {
    357   CriticalSectionScoped lock(crit_sect_.get());
    358   const int64_t now_ms = clock_->TimeInMilliseconds();
    359   PurgeOldFrameSamples(now_ms);
    360   UpdateSentBitrate(now_ms);
    361   return avg_sent_bit_rate_bps_;
    362 }
    363 
    364 VCMFrameCount MediaOptimization::SentFrameCount() {
    365   CriticalSectionScoped lock(crit_sect_.get());
    366   VCMFrameCount count;
    367   count.numDeltaFrames = delta_frame_cnt_;
    368   count.numKeyFrames = key_frame_cnt_;
    369   return count;
    370 }
    371 
    372 int32_t MediaOptimization::UpdateWithEncodedData(int encoded_length,
    373                                                  uint32_t timestamp,
    374                                                  FrameType encoded_frame_type) {
    375   CriticalSectionScoped lock(crit_sect_.get());
    376   const int64_t now_ms = clock_->TimeInMilliseconds();
    377   PurgeOldFrameSamples(now_ms);
    378   if (encoded_frame_samples_.size() > 0 &&
    379       encoded_frame_samples_.back().timestamp == timestamp) {
    380     // Frames having the same timestamp are generated from the same input
    381     // frame. We don't want to double count them, but only increment the
    382     // size_bytes.
    383     encoded_frame_samples_.back().size_bytes += encoded_length;
    384     encoded_frame_samples_.back().time_complete_ms = now_ms;
    385   } else {
    386     encoded_frame_samples_.push_back(
    387         EncodedFrameSample(encoded_length, timestamp, now_ms));
    388   }
    389   UpdateSentBitrate(now_ms);
    390   UpdateSentFramerate();
    391   if (encoded_length > 0) {
    392     const bool delta_frame = (encoded_frame_type != kVideoFrameKey);
    393 
    394     frame_dropper_->Fill(encoded_length, delta_frame);
    395     if (max_payload_size_ > 0 && encoded_length > 0) {
    396       const float min_packets_per_frame =
    397           encoded_length / static_cast<float>(max_payload_size_);
    398       if (delta_frame) {
    399         loss_prot_logic_->UpdatePacketsPerFrame(min_packets_per_frame,
    400                                                 clock_->TimeInMilliseconds());
    401       } else {
    402         loss_prot_logic_->UpdatePacketsPerFrameKey(
    403             min_packets_per_frame, clock_->TimeInMilliseconds());
    404       }
    405 
    406       if (enable_qm_) {
    407         // Update quality select with encoded length.
    408         qm_resolution_->UpdateEncodedSize(encoded_length, encoded_frame_type);
    409       }
    410     }
    411     if (!delta_frame && encoded_length > 0) {
    412       loss_prot_logic_->UpdateKeyFrameSize(static_cast<float>(encoded_length));
    413     }
    414 
    415     // Updating counters.
    416     if (delta_frame) {
    417       delta_frame_cnt_++;
    418     } else {
    419       key_frame_cnt_++;
    420     }
    421   }
    422 
    423   return VCM_OK;
    424 }
    425 
    426 void MediaOptimization::EnableQM(bool enable) {
    427   CriticalSectionScoped lock(crit_sect_.get());
    428   enable_qm_ = enable;
    429 }
    430 
    431 void MediaOptimization::EnableFrameDropper(bool enable) {
    432   CriticalSectionScoped lock(crit_sect_.get());
    433   frame_dropper_->Enable(enable);
    434 }
    435 
    436 void MediaOptimization::SuspendBelowMinBitrate(int threshold_bps,
    437                                                int window_bps) {
    438   CriticalSectionScoped lock(crit_sect_.get());
    439   assert(threshold_bps > 0 && window_bps >= 0);
    440   suspension_threshold_bps_ = threshold_bps;
    441   suspension_window_bps_ = window_bps;
    442   suspension_enabled_ = true;
    443   video_suspended_ = false;
    444 }
    445 
    446 bool MediaOptimization::IsVideoSuspended() const {
    447   CriticalSectionScoped lock(crit_sect_.get());
    448   return video_suspended_;
    449 }
    450 
    451 bool MediaOptimization::DropFrame() {
    452   CriticalSectionScoped lock(crit_sect_.get());
    453   UpdateIncomingFrameRate();
    454   // Leak appropriate number of bytes.
    455   frame_dropper_->Leak((uint32_t)(InputFrameRateInternal() + 0.5f));
    456   if (video_suspended_) {
    457     return true;  // Drop all frames when muted.
    458   }
    459   return frame_dropper_->DropFrame();
    460 }
    461 
    462 void MediaOptimization::UpdateContentData(
    463     const VideoContentMetrics* content_metrics) {
    464   CriticalSectionScoped lock(crit_sect_.get());
    465   // Updating content metrics.
    466   if (content_metrics == NULL) {
    467     // Disable QM if metrics are NULL.
    468     enable_qm_ = false;
    469     qm_resolution_->Reset();
    470   } else {
    471     content_->UpdateContentData(content_metrics);
    472   }
    473 }
    474 
    475 void MediaOptimization::UpdateIncomingFrameRate() {
    476   int64_t now = clock_->TimeInMilliseconds();
    477   if (incoming_frame_times_[0] == 0) {
    478     // No shifting if this is the first time.
    479   } else {
    480     // Shift all times one step.
    481     for (int32_t i = (kFrameCountHistorySize - 2); i >= 0; i--) {
    482       incoming_frame_times_[i + 1] = incoming_frame_times_[i];
    483     }
    484   }
    485   incoming_frame_times_[0] = now;
    486   ProcessIncomingFrameRate(now);
    487 }
    488 
    489 int32_t MediaOptimization::SelectQuality(
    490     VCMQMSettingsCallback* video_qmsettings_callback) {
    491   // Reset quantities for QM select.
    492   qm_resolution_->ResetQM();
    493 
    494   // Update QM will long-term averaged content metrics.
    495   qm_resolution_->UpdateContent(content_->LongTermAvgData());
    496 
    497   // Select quality mode.
    498   VCMResolutionScale* qm = NULL;
    499   int32_t ret = qm_resolution_->SelectResolution(&qm);
    500   if (ret < 0) {
    501     return ret;
    502   }
    503 
    504   // Check for updates to spatial/temporal modes.
    505   QMUpdate(qm, video_qmsettings_callback);
    506 
    507   // Reset all the rate and related frame counters quantities.
    508   qm_resolution_->ResetRates();
    509 
    510   // Reset counters.
    511   last_qm_update_time_ = clock_->TimeInMilliseconds();
    512 
    513   // Reset content metrics.
    514   content_->Reset();
    515 
    516   return VCM_OK;
    517 }
    518 
    519 void MediaOptimization::PurgeOldFrameSamples(int64_t now_ms) {
    520   while (!encoded_frame_samples_.empty()) {
    521     if (now_ms - encoded_frame_samples_.front().time_complete_ms >
    522         kBitrateAverageWinMs) {
    523       encoded_frame_samples_.pop_front();
    524     } else {
    525       break;
    526     }
    527   }
    528 }
    529 
    530 void MediaOptimization::UpdateSentBitrate(int64_t now_ms) {
    531   if (encoded_frame_samples_.empty()) {
    532     avg_sent_bit_rate_bps_ = 0;
    533     return;
    534   }
    535   int framesize_sum = 0;
    536   for (FrameSampleList::iterator it = encoded_frame_samples_.begin();
    537        it != encoded_frame_samples_.end();
    538        ++it) {
    539     framesize_sum += it->size_bytes;
    540   }
    541   float denom = static_cast<float>(
    542       now_ms - encoded_frame_samples_.front().time_complete_ms);
    543   if (denom >= 1.0f) {
    544     avg_sent_bit_rate_bps_ =
    545         static_cast<uint32_t>(framesize_sum * 8 * 1000 / denom + 0.5f);
    546   } else {
    547     avg_sent_bit_rate_bps_ = framesize_sum * 8;
    548   }
    549 }
    550 
    551 void MediaOptimization::UpdateSentFramerate() {
    552   if (encoded_frame_samples_.size() <= 1) {
    553     avg_sent_framerate_ = encoded_frame_samples_.size();
    554     return;
    555   }
    556   int denom = encoded_frame_samples_.back().timestamp -
    557               encoded_frame_samples_.front().timestamp;
    558   if (denom > 0) {
    559     avg_sent_framerate_ =
    560         (90000 * (encoded_frame_samples_.size() - 1) + denom / 2) / denom;
    561   } else {
    562     avg_sent_framerate_ = encoded_frame_samples_.size();
    563   }
    564 }
    565 
    566 bool MediaOptimization::QMUpdate(
    567     VCMResolutionScale* qm,
    568     VCMQMSettingsCallback* video_qmsettings_callback) {
    569   // Check for no change.
    570   if (!qm->change_resolution_spatial && !qm->change_resolution_temporal) {
    571     return false;
    572   }
    573 
    574   // Check for change in frame rate.
    575   if (qm->change_resolution_temporal) {
    576     incoming_frame_rate_ = qm->frame_rate;
    577     // Reset frame rate estimate.
    578     memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
    579   }
    580 
    581   // Check for change in frame size.
    582   if (qm->change_resolution_spatial) {
    583     codec_width_ = qm->codec_width;
    584     codec_height_ = qm->codec_height;
    585   }
    586 
    587   LOG(LS_INFO) << "Media optimizer requests the video resolution to be changed "
    588                   "to " << qm->codec_width << "x" << qm->codec_height << "@"
    589                << qm->frame_rate;
    590 
    591   // Update VPM with new target frame rate and frame size.
    592   // Note: use |qm->frame_rate| instead of |_incoming_frame_rate| for updating
    593   // target frame rate in VPM frame dropper. The quantity |_incoming_frame_rate|
    594   // will vary/fluctuate, and since we don't want to change the state of the
    595   // VPM frame dropper, unless a temporal action was selected, we use the
    596   // quantity |qm->frame_rate| for updating.
    597   video_qmsettings_callback->SetVideoQMSettings(
    598       qm->frame_rate, codec_width_, codec_height_);
    599   content_->UpdateFrameRate(qm->frame_rate);
    600   qm_resolution_->UpdateCodecParameters(
    601       qm->frame_rate, codec_width_, codec_height_);
    602   return true;
    603 }
    604 
    605 // Check timing constraints and look for significant change in:
    606 // (1) scene content,
    607 // (2) target bit rate.
    608 bool MediaOptimization::CheckStatusForQMchange() {
    609   bool status = true;
    610 
    611   // Check that we do not call QMSelect too often, and that we waited some time
    612   // (to sample the metrics) from the event last_change_time
    613   // last_change_time is the time where user changed the size/rate/frame rate
    614   // (via SetEncodingData).
    615   int64_t now = clock_->TimeInMilliseconds();
    616   if ((now - last_qm_update_time_) < kQmMinIntervalMs ||
    617       (now - last_change_time_) < kQmMinIntervalMs) {
    618     status = false;
    619   }
    620 
    621   return status;
    622 }
    623 
    624 // Allowing VCM to keep track of incoming frame rate.
    625 void MediaOptimization::ProcessIncomingFrameRate(int64_t now) {
    626   int32_t num = 0;
    627   int32_t nr_of_frames = 0;
    628   for (num = 1; num < (kFrameCountHistorySize - 1); ++num) {
    629     if (incoming_frame_times_[num] <= 0 ||
    630         // don't use data older than 2 s
    631         now - incoming_frame_times_[num] > kFrameHistoryWinMs) {
    632       break;
    633     } else {
    634       nr_of_frames++;
    635     }
    636   }
    637   if (num > 1) {
    638     const int64_t diff = now - incoming_frame_times_[num - 1];
    639     incoming_frame_rate_ = 1.0;
    640     if (diff > 0) {
    641       incoming_frame_rate_ = nr_of_frames * 1000.0f / static_cast<float>(diff);
    642     }
    643   }
    644 }
    645 
    646 void MediaOptimization::CheckSuspendConditions() {
    647   // Check conditions for SuspendBelowMinBitrate. |target_bit_rate_| is in bps.
    648   if (suspension_enabled_) {
    649     if (!video_suspended_) {
    650       // Check if we just went below the threshold.
    651       if (target_bit_rate_ < suspension_threshold_bps_) {
    652         video_suspended_ = true;
    653       }
    654     } else {
    655       // Video is already suspended. Check if we just went over the threshold
    656       // with a margin.
    657       if (target_bit_rate_ >
    658           suspension_threshold_bps_ + suspension_window_bps_) {
    659         video_suspended_ = false;
    660       }
    661     }
    662   }
    663 }
    664 
    665 }  // namespace media_optimization
    666 }  // namespace webrtc
    667