Home | History | Annotate | Download | only in video 
      1 /*
      2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/video/overuse_frame_detector.h"
     12 
     13 #include <assert.h>
     14 #include <math.h>
     15 
     16 #include <algorithm>
     17 #include <list>
     18 #include <map>
     19 
     20 #include "webrtc/base/checks.h"
     21 #include "webrtc/base/exp_filter.h"
     22 #include "webrtc/base/logging.h"
     23 #include "webrtc/system_wrappers/include/clock.h"
     24 
     25 namespace webrtc {
     26 
     27 namespace {
     28 const int64_t kProcessIntervalMs = 5000;
     29 
     30 // Delay between consecutive rampups. (Used for quick recovery.)
     31 const int kQuickRampUpDelayMs = 10 * 1000;
     32 // Delay between rampup attempts. Initially uses standard, scales up to max.
     33 const int kStandardRampUpDelayMs = 40 * 1000;
     34 const int kMaxRampUpDelayMs = 240 * 1000;
     35 // Expontential back-off factor, to prevent annoying up-down behaviour.
     36 const double kRampUpBackoffFactor = 2.0;
     37 
     38 // Max number of overuses detected before always applying the rampup delay.
     39 const int kMaxOverusesBeforeApplyRampupDelay = 4;
     40 
     41 // The maximum exponent to use in VCMExpFilter.
     42 const float kSampleDiffMs = 33.0f;
     43 const float kMaxExp = 7.0f;
     44 
     45 }  // namespace
     46 
     47 // Class for calculating the processing usage on the send-side (the average
     48 // processing time of a frame divided by the average time difference between
     49 // captured frames).
     50 class OveruseFrameDetector::SendProcessingUsage {
     51  public:
     52   explicit SendProcessingUsage(const CpuOveruseOptions& options)
     53       : kWeightFactorFrameDiff(0.998f),
     54         kWeightFactorProcessing(0.995f),
     55         kInitialSampleDiffMs(40.0f),
     56         kMaxSampleDiffMs(45.0f),
     57         count_(0),
     58         options_(options),
     59         filtered_processing_ms_(new rtc::ExpFilter(kWeightFactorProcessing)),
     60         filtered_frame_diff_ms_(new rtc::ExpFilter(kWeightFactorFrameDiff)) {
     61     Reset();
     62   }
     63   ~SendProcessingUsage() {}
     64 
     65   void Reset() {
     66     count_ = 0;
     67     filtered_frame_diff_ms_->Reset(kWeightFactorFrameDiff);
     68     filtered_frame_diff_ms_->Apply(1.0f, kInitialSampleDiffMs);
     69     filtered_processing_ms_->Reset(kWeightFactorProcessing);
     70     filtered_processing_ms_->Apply(1.0f, InitialProcessingMs());
     71   }
     72 
     73   void AddCaptureSample(float sample_ms) {
     74     float exp = sample_ms / kSampleDiffMs;
     75     exp = std::min(exp, kMaxExp);
     76     filtered_frame_diff_ms_->Apply(exp, sample_ms);
     77   }
     78 
     79   void AddSample(float processing_ms, int64_t diff_last_sample_ms) {
     80     ++count_;
     81     float exp = diff_last_sample_ms / kSampleDiffMs;
     82     exp = std::min(exp, kMaxExp);
     83     filtered_processing_ms_->Apply(exp, processing_ms);
     84   }
     85 
     86   int Value() const {
     87     if (count_ < static_cast<uint32_t>(options_.min_frame_samples)) {
     88       return static_cast<int>(InitialUsageInPercent() + 0.5f);
     89     }
     90     float frame_diff_ms = std::max(filtered_frame_diff_ms_->filtered(), 1.0f);
     91     frame_diff_ms = std::min(frame_diff_ms, kMaxSampleDiffMs);
     92     float encode_usage_percent =
     93         100.0f * filtered_processing_ms_->filtered() / frame_diff_ms;
     94     return static_cast<int>(encode_usage_percent + 0.5);
     95   }
     96 
     97  private:
     98   float InitialUsageInPercent() const {
     99     // Start in between the underuse and overuse threshold.
    100     return (options_.low_encode_usage_threshold_percent +
    101             options_.high_encode_usage_threshold_percent) / 2.0f;
    102   }
    103 
    104   float InitialProcessingMs() const {
    105     return InitialUsageInPercent() * kInitialSampleDiffMs / 100;
    106   }
    107 
    108   const float kWeightFactorFrameDiff;
    109   const float kWeightFactorProcessing;
    110   const float kInitialSampleDiffMs;
    111   const float kMaxSampleDiffMs;
    112   uint64_t count_;
    113   const CpuOveruseOptions options_;
    114   rtc::scoped_ptr<rtc::ExpFilter> filtered_processing_ms_;
    115   rtc::scoped_ptr<rtc::ExpFilter> filtered_frame_diff_ms_;
    116 };
    117 
    118 // Class for calculating the processing time of frames.
    119 class OveruseFrameDetector::FrameQueue {
    120  public:
    121   FrameQueue() : last_processing_time_ms_(-1) {}
    122   ~FrameQueue() {}
    123 
    124   // Called when a frame is captured.
    125   // Starts the measuring of the processing time of the frame.
    126   void Start(int64_t capture_time, int64_t now) {
    127     const size_t kMaxSize = 90;  // Allows for processing time of 1.5s at 60fps.
    128     if (frame_times_.size() > kMaxSize) {
    129       LOG(LS_WARNING) << "Max size reached, removed oldest frame.";
    130       frame_times_.erase(frame_times_.begin());
    131     }
    132     if (frame_times_.find(capture_time) != frame_times_.end()) {
    133       // Frame should not exist.
    134       assert(false);
    135       return;
    136     }
    137     frame_times_[capture_time] = now;
    138   }
    139 
    140   // Called when the processing of a frame has finished.
    141   // Returns the processing time of the frame.
    142   int End(int64_t capture_time, int64_t now) {
    143     std::map<int64_t, int64_t>::iterator it = frame_times_.find(capture_time);
    144     if (it == frame_times_.end()) {
    145       return -1;
    146     }
    147     // Remove any old frames up to current.
    148     // Old frames have been skipped by the capture process thread.
    149     // TODO(asapersson): Consider measuring time from first frame in list.
    150     last_processing_time_ms_ = now - (*it).second;
    151     frame_times_.erase(frame_times_.begin(), ++it);
    152     return last_processing_time_ms_;
    153   }
    154 
    155   void Reset() { frame_times_.clear(); }
    156   int NumFrames() const { return static_cast<int>(frame_times_.size()); }
    157   int last_processing_time_ms() const { return last_processing_time_ms_; }
    158 
    159  private:
    160   // Captured frames mapped by the capture time.
    161   std::map<int64_t, int64_t> frame_times_;
    162   int last_processing_time_ms_;
    163 };
    164 
    165 
    166 OveruseFrameDetector::OveruseFrameDetector(
    167     Clock* clock,
    168     const CpuOveruseOptions& options,
    169     CpuOveruseObserver* observer,
    170     CpuOveruseMetricsObserver* metrics_observer)
    171     : options_(options),
    172       observer_(observer),
    173       metrics_observer_(metrics_observer),
    174       clock_(clock),
    175       num_process_times_(0),
    176       last_capture_time_(0),
    177       num_pixels_(0),
    178       next_process_time_(clock_->TimeInMilliseconds()),
    179       last_overuse_time_(0),
    180       checks_above_threshold_(0),
    181       num_overuse_detections_(0),
    182       last_rampup_time_(0),
    183       in_quick_rampup_(false),
    184       current_rampup_delay_ms_(kStandardRampUpDelayMs),
    185       last_sample_time_ms_(0),
    186       usage_(new SendProcessingUsage(options)),
    187       frame_queue_(new FrameQueue()) {
    188   RTC_DCHECK(metrics_observer != nullptr);
    189   // Make sure stats are initially up-to-date. This simplifies unit testing
    190   // since we don't have to trigger an update using one of the methods which
    191   // would also alter the overuse state.
    192   UpdateCpuOveruseMetrics();
    193   processing_thread_.DetachFromThread();
    194 }
    195 
    196 OveruseFrameDetector::~OveruseFrameDetector() {
    197 }
    198 
    199 int OveruseFrameDetector::LastProcessingTimeMs() const {
    200   rtc::CritScope cs(&crit_);
    201   return frame_queue_->last_processing_time_ms();
    202 }
    203 
    204 int OveruseFrameDetector::FramesInQueue() const {
    205   rtc::CritScope cs(&crit_);
    206   return frame_queue_->NumFrames();
    207 }
    208 
    209 void OveruseFrameDetector::UpdateCpuOveruseMetrics() {
    210   metrics_.encode_usage_percent = usage_->Value();
    211 
    212   metrics_observer_->CpuOveruseMetricsUpdated(metrics_);
    213 }
    214 
    215 int64_t OveruseFrameDetector::TimeUntilNextProcess() {
    216   RTC_DCHECK(processing_thread_.CalledOnValidThread());
    217   return next_process_time_ - clock_->TimeInMilliseconds();
    218 }
    219 
    220 bool OveruseFrameDetector::FrameSizeChanged(int num_pixels) const {
    221   if (num_pixels != num_pixels_) {
    222     return true;
    223   }
    224   return false;
    225 }
    226 
    227 bool OveruseFrameDetector::FrameTimeoutDetected(int64_t now) const {
    228   if (last_capture_time_ == 0) {
    229     return false;
    230   }
    231   return (now - last_capture_time_) > options_.frame_timeout_interval_ms;
    232 }
    233 
    234 void OveruseFrameDetector::ResetAll(int num_pixels) {
    235   num_pixels_ = num_pixels;
    236   usage_->Reset();
    237   frame_queue_->Reset();
    238   last_capture_time_ = 0;
    239   num_process_times_ = 0;
    240   UpdateCpuOveruseMetrics();
    241 }
    242 
    243 void OveruseFrameDetector::FrameCaptured(int width,
    244                                          int height,
    245                                          int64_t capture_time_ms) {
    246   rtc::CritScope cs(&crit_);
    247 
    248   int64_t now = clock_->TimeInMilliseconds();
    249   if (FrameSizeChanged(width * height) || FrameTimeoutDetected(now)) {
    250     ResetAll(width * height);
    251   }
    252 
    253   if (last_capture_time_ != 0)
    254     usage_->AddCaptureSample(now - last_capture_time_);
    255 
    256   last_capture_time_ = now;
    257 
    258   frame_queue_->Start(capture_time_ms, now);
    259 }
    260 
    261 void OveruseFrameDetector::FrameSent(int64_t capture_time_ms) {
    262   rtc::CritScope cs(&crit_);
    263   int delay_ms = frame_queue_->End(capture_time_ms,
    264                                    clock_->TimeInMilliseconds());
    265   if (delay_ms > 0) {
    266     AddProcessingTime(delay_ms);
    267   }
    268 }
    269 
    270 void OveruseFrameDetector::AddProcessingTime(int elapsed_ms) {
    271   int64_t now = clock_->TimeInMilliseconds();
    272   if (last_sample_time_ms_ != 0) {
    273     int64_t diff_ms = now - last_sample_time_ms_;
    274     usage_->AddSample(elapsed_ms, diff_ms);
    275   }
    276   last_sample_time_ms_ = now;
    277   UpdateCpuOveruseMetrics();
    278 }
    279 
    280 int32_t OveruseFrameDetector::Process() {
    281   RTC_DCHECK(processing_thread_.CalledOnValidThread());
    282 
    283   int64_t now = clock_->TimeInMilliseconds();
    284 
    285   // Used to protect against Process() being called too often.
    286   if (now < next_process_time_)
    287     return 0;
    288 
    289   next_process_time_ = now + kProcessIntervalMs;
    290 
    291   CpuOveruseMetrics current_metrics;
    292   {
    293     rtc::CritScope cs(&crit_);
    294     ++num_process_times_;
    295 
    296     current_metrics = metrics_;
    297     if (num_process_times_ <= options_.min_process_count)
    298       return 0;
    299   }
    300 
    301   if (IsOverusing(current_metrics)) {
    302     // If the last thing we did was going up, and now have to back down, we need
    303     // to check if this peak was short. If so we should back off to avoid going
    304     // back and forth between this load, the system doesn't seem to handle it.
    305     bool check_for_backoff = last_rampup_time_ > last_overuse_time_;
    306     if (check_for_backoff) {
    307       if (now - last_rampup_time_ < kStandardRampUpDelayMs ||
    308           num_overuse_detections_ > kMaxOverusesBeforeApplyRampupDelay) {
    309         // Going up was not ok for very long, back off.
    310         current_rampup_delay_ms_ *= kRampUpBackoffFactor;
    311         if (current_rampup_delay_ms_ > kMaxRampUpDelayMs)
    312           current_rampup_delay_ms_ = kMaxRampUpDelayMs;
    313       } else {
    314         // Not currently backing off, reset rampup delay.
    315         current_rampup_delay_ms_ = kStandardRampUpDelayMs;
    316       }
    317     }
    318 
    319     last_overuse_time_ = now;
    320     in_quick_rampup_ = false;
    321     checks_above_threshold_ = 0;
    322     ++num_overuse_detections_;
    323 
    324     if (observer_ != NULL)
    325       observer_->OveruseDetected();
    326   } else if (IsUnderusing(current_metrics, now)) {
    327     last_rampup_time_ = now;
    328     in_quick_rampup_ = true;
    329 
    330     if (observer_ != NULL)
    331       observer_->NormalUsage();
    332   }
    333 
    334   int rampup_delay =
    335       in_quick_rampup_ ? kQuickRampUpDelayMs : current_rampup_delay_ms_;
    336 
    337   LOG(LS_VERBOSE) << " Frame stats: "
    338                   << " encode usage " << current_metrics.encode_usage_percent
    339                   << " overuse detections " << num_overuse_detections_
    340                   << " rampup delay " << rampup_delay;
    341 
    342   return 0;
    343 }
    344 
    345 bool OveruseFrameDetector::IsOverusing(const CpuOveruseMetrics& metrics) {
    346   if (metrics.encode_usage_percent >=
    347       options_.high_encode_usage_threshold_percent) {
    348     ++checks_above_threshold_;
    349   } else {
    350     checks_above_threshold_ = 0;
    351   }
    352   return checks_above_threshold_ >= options_.high_threshold_consecutive_count;
    353 }
    354 
    355 bool OveruseFrameDetector::IsUnderusing(const CpuOveruseMetrics& metrics,
    356                                         int64_t time_now) {
    357   int delay = in_quick_rampup_ ? kQuickRampUpDelayMs : current_rampup_delay_ms_;
    358   if (time_now < last_rampup_time_ + delay)
    359     return false;
    360 
    361   return metrics.encode_usage_percent <
    362          options_.low_encode_usage_threshold_percent;
    363 }
    364 }  // namespace webrtc
    365