Home | History | Annotate | Download | only in video_coding 
      1 /*
      2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/modules/video_coding/media_optimization.h"
     12 
     13 #include "webrtc/base/logging.h"
     14 #include "webrtc/modules/video_coding/content_metrics_processing.h"
     15 #include "webrtc/modules/video_coding/qm_select.h"
     16 #include "webrtc/modules/video_coding/utility/frame_dropper.h"
     17 #include "webrtc/system_wrappers/include/clock.h"
     18 
     19 namespace webrtc {
     20 namespace media_optimization {
     21 namespace {
     22 void UpdateProtectionCallback(
     23     VCMProtectionMethod* selected_method,
     24     uint32_t* video_rate_bps,
     25     uint32_t* nack_overhead_rate_bps,
     26     uint32_t* fec_overhead_rate_bps,
     27     VCMProtectionCallback* video_protection_callback) {
     28   FecProtectionParams delta_fec_params;
     29   FecProtectionParams key_fec_params;
     30   // Get the FEC code rate for Key frames (set to 0 when NA).
     31   key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK();
     32 
     33   // Get the FEC code rate for Delta frames (set to 0 when NA).
     34   delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD();
     35 
     36   // Get the FEC-UEP protection status for Key frames: UEP on/off.
     37   key_fec_params.use_uep_protection = selected_method->RequiredUepProtectionK();
     38 
     39   // Get the FEC-UEP protection status for Delta frames: UEP on/off.
     40   delta_fec_params.use_uep_protection =
     41       selected_method->RequiredUepProtectionD();
     42 
     43   // The RTP module currently requires the same |max_fec_frames| for both
     44   // key and delta frames.
     45   delta_fec_params.max_fec_frames = selected_method->MaxFramesFec();
     46   key_fec_params.max_fec_frames = selected_method->MaxFramesFec();
     47 
     48   // Set the FEC packet mask type. |kFecMaskBursty| is more effective for
     49   // consecutive losses and little/no packet re-ordering. As we currently
     50   // do not have feedback data on the degree of correlated losses and packet
     51   // re-ordering, we keep default setting to |kFecMaskRandom| for now.
     52   delta_fec_params.fec_mask_type = kFecMaskRandom;
     53   key_fec_params.fec_mask_type = kFecMaskRandom;
     54 
     55   // TODO(Marco): Pass FEC protection values per layer.
     56   video_protection_callback->ProtectionRequest(
     57       &delta_fec_params, &key_fec_params, video_rate_bps,
     58       nack_overhead_rate_bps, fec_overhead_rate_bps);
     59 }
     60 }  // namespace
     61 
     62 struct MediaOptimization::EncodedFrameSample {
     63   EncodedFrameSample(size_t size_bytes,
     64                      uint32_t timestamp,
     65                      int64_t time_complete_ms)
     66       : size_bytes(size_bytes),
     67         timestamp(timestamp),
     68         time_complete_ms(time_complete_ms) {}
     69 
     70   size_t size_bytes;
     71   uint32_t timestamp;
     72   int64_t time_complete_ms;
     73 };
     74 
     75 MediaOptimization::MediaOptimization(Clock* clock)
     76     : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
     77       clock_(clock),
     78       max_bit_rate_(0),
     79       send_codec_type_(kVideoCodecUnknown),
     80       codec_width_(0),
     81       codec_height_(0),
     82       user_frame_rate_(0),
     83       frame_dropper_(new FrameDropper),
     84       loss_prot_logic_(
     85           new VCMLossProtectionLogic(clock_->TimeInMilliseconds())),
     86       fraction_lost_(0),
     87       send_statistics_zero_encode_(0),
     88       max_payload_size_(1460),
     89       video_target_bitrate_(0),
     90       incoming_frame_rate_(0),
     91       enable_qm_(false),
     92       encoded_frame_samples_(),
     93       avg_sent_bit_rate_bps_(0),
     94       avg_sent_framerate_(0),
     95       key_frame_cnt_(0),
     96       delta_frame_cnt_(0),
     97       content_(new VCMContentMetricsProcessing()),
     98       qm_resolution_(new VCMQmResolution()),
     99       last_qm_update_time_(0),
    100       last_change_time_(0),
    101       num_layers_(0),
    102       suspension_enabled_(false),
    103       video_suspended_(false),
    104       suspension_threshold_bps_(0),
    105       suspension_window_bps_(0) {
    106   memset(send_statistics_, 0, sizeof(send_statistics_));
    107   memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
    108 }
    109 
    110 MediaOptimization::~MediaOptimization(void) {
    111   loss_prot_logic_->Release();
    112 }
    113 
    114 void MediaOptimization::Reset() {
    115   CriticalSectionScoped lock(crit_sect_.get());
    116   SetEncodingDataInternal(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0,
    117                           max_payload_size_);
    118   memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
    119   incoming_frame_rate_ = 0.0;
    120   frame_dropper_->Reset();
    121   loss_prot_logic_->Reset(clock_->TimeInMilliseconds());
    122   frame_dropper_->SetRates(0, 0);
    123   content_->Reset();
    124   qm_resolution_->Reset();
    125   loss_prot_logic_->UpdateFrameRate(incoming_frame_rate_);
    126   loss_prot_logic_->Reset(clock_->TimeInMilliseconds());
    127   send_statistics_zero_encode_ = 0;
    128   video_target_bitrate_ = 0;
    129   codec_width_ = 0;
    130   codec_height_ = 0;
    131   user_frame_rate_ = 0;
    132   key_frame_cnt_ = 0;
    133   delta_frame_cnt_ = 0;
    134   last_qm_update_time_ = 0;
    135   last_change_time_ = 0;
    136   encoded_frame_samples_.clear();
    137   avg_sent_bit_rate_bps_ = 0;
    138   num_layers_ = 1;
    139 }
    140 
    141 void MediaOptimization::SetEncodingData(VideoCodecType send_codec_type,
    142                                         int32_t max_bit_rate,
    143                                         uint32_t target_bitrate,
    144                                         uint16_t width,
    145                                         uint16_t height,
    146                                         uint32_t frame_rate,
    147                                         int num_layers,
    148                                         int32_t mtu) {
    149   CriticalSectionScoped lock(crit_sect_.get());
    150   SetEncodingDataInternal(send_codec_type, max_bit_rate, frame_rate,
    151                           target_bitrate, width, height, num_layers, mtu);
    152 }
    153 
    154 void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type,
    155                                                 int32_t max_bit_rate,
    156                                                 uint32_t frame_rate,
    157                                                 uint32_t target_bitrate,
    158                                                 uint16_t width,
    159                                                 uint16_t height,
    160                                                 int num_layers,
    161                                                 int32_t mtu) {
    162   // Everything codec specific should be reset here since this means the codec
    163   // has changed. If native dimension values have changed, then either user
    164   // initiated change, or QM initiated change. Will be able to determine only
    165   // after the processing of the first frame.
    166   last_change_time_ = clock_->TimeInMilliseconds();
    167   content_->Reset();
    168   content_->UpdateFrameRate(frame_rate);
    169 
    170   max_bit_rate_ = max_bit_rate;
    171   send_codec_type_ = send_codec_type;
    172   video_target_bitrate_ = target_bitrate;
    173   float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
    174   loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
    175   loss_prot_logic_->UpdateFrameRate(static_cast<float>(frame_rate));
    176   loss_prot_logic_->UpdateFrameSize(width, height);
    177   loss_prot_logic_->UpdateNumLayers(num_layers);
    178   frame_dropper_->Reset();
    179   frame_dropper_->SetRates(target_bitrate_kbps, static_cast<float>(frame_rate));
    180   user_frame_rate_ = static_cast<float>(frame_rate);
    181   codec_width_ = width;
    182   codec_height_ = height;
    183   num_layers_ = (num_layers <= 1) ? 1 : num_layers;  // Can also be zero.
    184   max_payload_size_ = mtu;
    185   qm_resolution_->Initialize(target_bitrate_kbps, user_frame_rate_,
    186                              codec_width_, codec_height_, num_layers_);
    187 }
    188 
    189 uint32_t MediaOptimization::SetTargetRates(
    190     uint32_t target_bitrate,
    191     uint8_t fraction_lost,
    192     int64_t round_trip_time_ms,
    193     VCMProtectionCallback* protection_callback,
    194     VCMQMSettingsCallback* qmsettings_callback) {
    195   CriticalSectionScoped lock(crit_sect_.get());
    196   VCMProtectionMethod* selected_method = loss_prot_logic_->SelectedMethod();
    197   float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
    198   loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
    199   loss_prot_logic_->UpdateRtt(round_trip_time_ms);
    200 
    201   // Get frame rate for encoder: this is the actual/sent frame rate.
    202   float actual_frame_rate = SentFrameRateInternal();
    203 
    204   // Sanity check.
    205   if (actual_frame_rate < 1.0) {
    206     actual_frame_rate = 1.0;
    207   }
    208 
    209   // Update frame rate for the loss protection logic class: frame rate should
    210   // be the actual/sent rate.
    211   loss_prot_logic_->UpdateFrameRate(actual_frame_rate);
    212 
    213   fraction_lost_ = fraction_lost;
    214 
    215   // Returns the filtered packet loss, used for the protection setting.
    216   // The filtered loss may be the received loss (no filter), or some
    217   // filtered value (average or max window filter).
    218   // Use max window filter for now.
    219   FilterPacketLossMode filter_mode = kMaxFilter;
    220   uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss(
    221       clock_->TimeInMilliseconds(), filter_mode, fraction_lost);
    222 
    223   // For now use the filtered loss for computing the robustness settings.
    224   loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc);
    225 
    226   // Rate cost of the protection methods.
    227   float protection_overhead_rate = 0.0f;
    228 
    229   // Update protection settings, when applicable.
    230   float sent_video_rate_kbps = 0.0f;
    231   if (loss_prot_logic_->SelectedType() != kNone) {
    232     // Update protection method with content metrics.
    233     selected_method->UpdateContentMetrics(content_->ShortTermAvgData());
    234 
    235     // Update method will compute the robustness settings for the given
    236     // protection method and the overhead cost
    237     // the protection method is set by the user via SetVideoProtection.
    238     loss_prot_logic_->UpdateMethod();
    239 
    240     // Update protection callback with protection settings.
    241     uint32_t sent_video_rate_bps = 0;
    242     uint32_t sent_nack_rate_bps = 0;
    243     uint32_t sent_fec_rate_bps = 0;
    244     // Get the bit cost of protection method, based on the amount of
    245     // overhead data actually transmitted (including headers) the last
    246     // second.
    247     if (protection_callback) {
    248       UpdateProtectionCallback(selected_method, &sent_video_rate_bps,
    249                                &sent_nack_rate_bps, &sent_fec_rate_bps,
    250                                protection_callback);
    251     }
    252     uint32_t sent_total_rate_bps =
    253         sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps;
    254     // Estimate the overhead costs of the next second as staying the same
    255     // wrt the source bitrate.
    256     if (sent_total_rate_bps > 0) {
    257       protection_overhead_rate =
    258           static_cast<float>(sent_nack_rate_bps + sent_fec_rate_bps) /
    259           sent_total_rate_bps;
    260     }
    261     // Cap the overhead estimate to 50%.
    262     if (protection_overhead_rate > 0.5)
    263       protection_overhead_rate = 0.5;
    264 
    265     // Get the effective packet loss for encoder ER when applicable. Should be
    266     // passed to encoder via fraction_lost.
    267     packet_loss_enc = selected_method->RequiredPacketLossER();
    268     sent_video_rate_kbps = static_cast<float>(sent_video_rate_bps) / 1000.0f;
    269   }
    270 
    271   // Source coding rate: total rate - protection overhead.
    272   video_target_bitrate_ = target_bitrate * (1.0 - protection_overhead_rate);
    273 
    274   // Cap target video bitrate to codec maximum.
    275   if (max_bit_rate_ > 0 && video_target_bitrate_ > max_bit_rate_) {
    276     video_target_bitrate_ = max_bit_rate_;
    277   }
    278 
    279   // Update encoding rates following protection settings.
    280   float target_video_bitrate_kbps =
    281       static_cast<float>(video_target_bitrate_) / 1000.0f;
    282   frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_);
    283 
    284   if (enable_qm_ && qmsettings_callback) {
    285     // Update QM with rates.
    286     qm_resolution_->UpdateRates(target_video_bitrate_kbps, sent_video_rate_kbps,
    287                                 incoming_frame_rate_, fraction_lost_);
    288     // Check for QM selection.
    289     bool select_qm = CheckStatusForQMchange();
    290     if (select_qm) {
    291       SelectQuality(qmsettings_callback);
    292     }
    293     // Reset the short-term averaged content data.
    294     content_->ResetShortTermAvgData();
    295   }
    296 
    297   CheckSuspendConditions();
    298 
    299   return video_target_bitrate_;
    300 }
    301 
    302 void MediaOptimization::SetProtectionMethod(VCMProtectionMethodEnum method) {
    303   CriticalSectionScoped lock(crit_sect_.get());
    304   loss_prot_logic_->SetMethod(method);
    305 }
    306 
    307 uint32_t MediaOptimization::InputFrameRate() {
    308   CriticalSectionScoped lock(crit_sect_.get());
    309   return InputFrameRateInternal();
    310 }
    311 
    312 uint32_t MediaOptimization::InputFrameRateInternal() {
    313   ProcessIncomingFrameRate(clock_->TimeInMilliseconds());
    314   return uint32_t(incoming_frame_rate_ + 0.5f);
    315 }
    316 
    317 uint32_t MediaOptimization::SentFrameRate() {
    318   CriticalSectionScoped lock(crit_sect_.get());
    319   return SentFrameRateInternal();
    320 }
    321 
    322 uint32_t MediaOptimization::SentFrameRateInternal() {
    323   PurgeOldFrameSamples(clock_->TimeInMilliseconds());
    324   UpdateSentFramerate();
    325   return avg_sent_framerate_;
    326 }
    327 
    328 uint32_t MediaOptimization::SentBitRate() {
    329   CriticalSectionScoped lock(crit_sect_.get());
    330   const int64_t now_ms = clock_->TimeInMilliseconds();
    331   PurgeOldFrameSamples(now_ms);
    332   UpdateSentBitrate(now_ms);
    333   return avg_sent_bit_rate_bps_;
    334 }
    335 
    336 int32_t MediaOptimization::UpdateWithEncodedData(
    337     const EncodedImage& encoded_image) {
    338   size_t encoded_length = encoded_image._length;
    339   uint32_t timestamp = encoded_image._timeStamp;
    340   CriticalSectionScoped lock(crit_sect_.get());
    341   const int64_t now_ms = clock_->TimeInMilliseconds();
    342   PurgeOldFrameSamples(now_ms);
    343   if (encoded_frame_samples_.size() > 0 &&
    344       encoded_frame_samples_.back().timestamp == timestamp) {
    345     // Frames having the same timestamp are generated from the same input
    346     // frame. We don't want to double count them, but only increment the
    347     // size_bytes.
    348     encoded_frame_samples_.back().size_bytes += encoded_length;
    349     encoded_frame_samples_.back().time_complete_ms = now_ms;
    350   } else {
    351     encoded_frame_samples_.push_back(
    352         EncodedFrameSample(encoded_length, timestamp, now_ms));
    353   }
    354   UpdateSentBitrate(now_ms);
    355   UpdateSentFramerate();
    356   if (encoded_length > 0) {
    357     const bool delta_frame = encoded_image._frameType != kVideoFrameKey;
    358 
    359     frame_dropper_->Fill(encoded_length, delta_frame);
    360     if (max_payload_size_ > 0 && encoded_length > 0) {
    361       const float min_packets_per_frame =
    362           encoded_length / static_cast<float>(max_payload_size_);
    363       if (delta_frame) {
    364         loss_prot_logic_->UpdatePacketsPerFrame(min_packets_per_frame,
    365                                                 clock_->TimeInMilliseconds());
    366       } else {
    367         loss_prot_logic_->UpdatePacketsPerFrameKey(
    368             min_packets_per_frame, clock_->TimeInMilliseconds());
    369       }
    370 
    371       if (enable_qm_) {
    372         // Update quality select with encoded length.
    373         qm_resolution_->UpdateEncodedSize(encoded_length);
    374       }
    375     }
    376     if (!delta_frame && encoded_length > 0) {
    377       loss_prot_logic_->UpdateKeyFrameSize(static_cast<float>(encoded_length));
    378     }
    379 
    380     // Updating counters.
    381     if (delta_frame) {
    382       delta_frame_cnt_++;
    383     } else {
    384       key_frame_cnt_++;
    385     }
    386   }
    387 
    388   return VCM_OK;
    389 }
    390 
    391 void MediaOptimization::EnableQM(bool enable) {
    392   CriticalSectionScoped lock(crit_sect_.get());
    393   enable_qm_ = enable;
    394 }
    395 
    396 void MediaOptimization::EnableFrameDropper(bool enable) {
    397   CriticalSectionScoped lock(crit_sect_.get());
    398   frame_dropper_->Enable(enable);
    399 }
    400 
    401 void MediaOptimization::SuspendBelowMinBitrate(int threshold_bps,
    402                                                int window_bps) {
    403   CriticalSectionScoped lock(crit_sect_.get());
    404   assert(threshold_bps > 0 && window_bps >= 0);
    405   suspension_threshold_bps_ = threshold_bps;
    406   suspension_window_bps_ = window_bps;
    407   suspension_enabled_ = true;
    408   video_suspended_ = false;
    409 }
    410 
    411 bool MediaOptimization::IsVideoSuspended() const {
    412   CriticalSectionScoped lock(crit_sect_.get());
    413   return video_suspended_;
    414 }
    415 
    416 bool MediaOptimization::DropFrame() {
    417   CriticalSectionScoped lock(crit_sect_.get());
    418   UpdateIncomingFrameRate();
    419   // Leak appropriate number of bytes.
    420   frame_dropper_->Leak((uint32_t)(InputFrameRateInternal() + 0.5f));
    421   if (video_suspended_) {
    422     return true;  // Drop all frames when muted.
    423   }
    424   return frame_dropper_->DropFrame();
    425 }
    426 
    427 void MediaOptimization::UpdateContentData(
    428     const VideoContentMetrics* content_metrics) {
    429   CriticalSectionScoped lock(crit_sect_.get());
    430   // Updating content metrics.
    431   if (content_metrics == NULL) {
    432     // Disable QM if metrics are NULL.
    433     enable_qm_ = false;
    434     qm_resolution_->Reset();
    435   } else {
    436     content_->UpdateContentData(content_metrics);
    437   }
    438 }
    439 
    440 void MediaOptimization::UpdateIncomingFrameRate() {
    441   int64_t now = clock_->TimeInMilliseconds();
    442   if (incoming_frame_times_[0] == 0) {
    443     // No shifting if this is the first time.
    444   } else {
    445     // Shift all times one step.
    446     for (int32_t i = (kFrameCountHistorySize - 2); i >= 0; i--) {
    447       incoming_frame_times_[i + 1] = incoming_frame_times_[i];
    448     }
    449   }
    450   incoming_frame_times_[0] = now;
    451   ProcessIncomingFrameRate(now);
    452 }
    453 
    454 int32_t MediaOptimization::SelectQuality(
    455     VCMQMSettingsCallback* video_qmsettings_callback) {
    456   // Reset quantities for QM select.
    457   qm_resolution_->ResetQM();
    458 
    459   // Update QM will long-term averaged content metrics.
    460   qm_resolution_->UpdateContent(content_->LongTermAvgData());
    461 
    462   // Select quality mode.
    463   VCMResolutionScale* qm = NULL;
    464   int32_t ret = qm_resolution_->SelectResolution(&qm);
    465   if (ret < 0) {
    466     return ret;
    467   }
    468 
    469   // Check for updates to spatial/temporal modes.
    470   QMUpdate(qm, video_qmsettings_callback);
    471 
    472   // Reset all the rate and related frame counters quantities.
    473   qm_resolution_->ResetRates();
    474 
    475   // Reset counters.
    476   last_qm_update_time_ = clock_->TimeInMilliseconds();
    477 
    478   // Reset content metrics.
    479   content_->Reset();
    480 
    481   return VCM_OK;
    482 }
    483 
    484 void MediaOptimization::PurgeOldFrameSamples(int64_t now_ms) {
    485   while (!encoded_frame_samples_.empty()) {
    486     if (now_ms - encoded_frame_samples_.front().time_complete_ms >
    487         kBitrateAverageWinMs) {
    488       encoded_frame_samples_.pop_front();
    489     } else {
    490       break;
    491     }
    492   }
    493 }
    494 
    495 void MediaOptimization::UpdateSentBitrate(int64_t now_ms) {
    496   if (encoded_frame_samples_.empty()) {
    497     avg_sent_bit_rate_bps_ = 0;
    498     return;
    499   }
    500   size_t framesize_sum = 0;
    501   for (FrameSampleList::iterator it = encoded_frame_samples_.begin();
    502        it != encoded_frame_samples_.end(); ++it) {
    503     framesize_sum += it->size_bytes;
    504   }
    505   float denom = static_cast<float>(
    506       now_ms - encoded_frame_samples_.front().time_complete_ms);
    507   if (denom >= 1.0f) {
    508     avg_sent_bit_rate_bps_ =
    509         static_cast<uint32_t>(framesize_sum * 8.0f * 1000.0f / denom + 0.5f);
    510   } else {
    511     avg_sent_bit_rate_bps_ = framesize_sum * 8;
    512   }
    513 }
    514 
    515 void MediaOptimization::UpdateSentFramerate() {
    516   if (encoded_frame_samples_.size() <= 1) {
    517     avg_sent_framerate_ = encoded_frame_samples_.size();
    518     return;
    519   }
    520   int denom = encoded_frame_samples_.back().timestamp -
    521               encoded_frame_samples_.front().timestamp;
    522   if (denom > 0) {
    523     avg_sent_framerate_ =
    524         (90000 * (encoded_frame_samples_.size() - 1) + denom / 2) / denom;
    525   } else {
    526     avg_sent_framerate_ = encoded_frame_samples_.size();
    527   }
    528 }
    529 
    530 bool MediaOptimization::QMUpdate(
    531     VCMResolutionScale* qm,
    532     VCMQMSettingsCallback* video_qmsettings_callback) {
    533   // Check for no change.
    534   if (!qm->change_resolution_spatial && !qm->change_resolution_temporal) {
    535     return false;
    536   }
    537 
    538   // Check for change in frame rate.
    539   if (qm->change_resolution_temporal) {
    540     incoming_frame_rate_ = qm->frame_rate;
    541     // Reset frame rate estimate.
    542     memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
    543   }
    544 
    545   // Check for change in frame size.
    546   if (qm->change_resolution_spatial) {
    547     codec_width_ = qm->codec_width;
    548     codec_height_ = qm->codec_height;
    549   }
    550 
    551   LOG(LS_INFO) << "Media optimizer requests the video resolution to be changed "
    552                   "to "
    553                << qm->codec_width << "x" << qm->codec_height << "@"
    554                << qm->frame_rate;
    555 
    556   // Update VPM with new target frame rate and frame size.
    557   // Note: use |qm->frame_rate| instead of |_incoming_frame_rate| for updating
    558   // target frame rate in VPM frame dropper. The quantity |_incoming_frame_rate|
    559   // will vary/fluctuate, and since we don't want to change the state of the
    560   // VPM frame dropper, unless a temporal action was selected, we use the
    561   // quantity |qm->frame_rate| for updating.
    562   video_qmsettings_callback->SetVideoQMSettings(qm->frame_rate, codec_width_,
    563                                                 codec_height_);
    564   content_->UpdateFrameRate(qm->frame_rate);
    565   qm_resolution_->UpdateCodecParameters(qm->frame_rate, codec_width_,
    566                                         codec_height_);
    567   return true;
    568 }
    569 
    570 // Check timing constraints and look for significant change in:
    571 // (1) scene content,
    572 // (2) target bit rate.
    573 bool MediaOptimization::CheckStatusForQMchange() {
    574   bool status = true;
    575 
    576   // Check that we do not call QMSelect too often, and that we waited some time
    577   // (to sample the metrics) from the event last_change_time
    578   // last_change_time is the time where user changed the size/rate/frame rate
    579   // (via SetEncodingData).
    580   int64_t now = clock_->TimeInMilliseconds();
    581   if ((now - last_qm_update_time_) < kQmMinIntervalMs ||
    582       (now - last_change_time_) < kQmMinIntervalMs) {
    583     status = false;
    584   }
    585 
    586   return status;
    587 }
    588 
    589 // Allowing VCM to keep track of incoming frame rate.
    590 void MediaOptimization::ProcessIncomingFrameRate(int64_t now) {
    591   int32_t num = 0;
    592   int32_t nr_of_frames = 0;
    593   for (num = 1; num < (kFrameCountHistorySize - 1); ++num) {
    594     if (incoming_frame_times_[num] <= 0 ||
    595         // don't use data older than 2 s
    596         now - incoming_frame_times_[num] > kFrameHistoryWinMs) {
    597       break;
    598     } else {
    599       nr_of_frames++;
    600     }
    601   }
    602   if (num > 1) {
    603     const int64_t diff =
    604         incoming_frame_times_[0] - incoming_frame_times_[num - 1];
    605     incoming_frame_rate_ = 0.0;  // No frame rate estimate available.
    606     if (diff > 0) {
    607       incoming_frame_rate_ = nr_of_frames * 1000.0f / static_cast<float>(diff);
    608     }
    609   }
    610 }
    611 
    612 void MediaOptimization::CheckSuspendConditions() {
    613   // Check conditions for SuspendBelowMinBitrate. |video_target_bitrate_| is in
    614   // bps.
    615   if (suspension_enabled_) {
    616     if (!video_suspended_) {
    617       // Check if we just went below the threshold.
    618       if (video_target_bitrate_ < suspension_threshold_bps_) {
    619         video_suspended_ = true;
    620       }
    621     } else {
    622       // Video is already suspended. Check if we just went over the threshold
    623       // with a margin.
    624       if (video_target_bitrate_ >
    625           suspension_threshold_bps_ + suspension_window_bps_) {
    626         video_suspended_ = false;
    627       }
    628     }
    629   }
    630 }
    631 
    632 }  // namespace media_optimization
    633 }  // namespace webrtc
    634