Home | History | Annotate | Download | only in utility 
      1 /*
      2  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/modules/video_coding/utility/quality_scaler.h"
     12 
     13 #include "testing/gtest/include/gtest/gtest.h"
     14 
     15 namespace webrtc {
     16 namespace {
     17 static const int kNumSeconds = 10;
     18 static const int kWidth = 1920;
     19 static const int kHalfWidth = kWidth / 2;
     20 static const int kHeight = 1080;
     21 static const int kFramerate = 30;
     22 static const int kLowQp = 15;
     23 static const int kNormalQp = 30;
     24 static const int kHighQp = 40;
     25 static const int kMaxQp = 56;
     26 }  // namespace
     27 
     28 class QualityScalerTest : public ::testing::Test {
     29  public:
     30   // Temporal and spatial resolution.
     31   struct Resolution {
     32     int framerate;
     33     int width;
     34     int height;
     35   };
     36 
     37  protected:
     38   enum ScaleDirection {
     39     kKeepScaleAtHighQp,
     40     kScaleDown,
     41     kScaleDownAboveHighQp,
     42     kScaleUp
     43   };
     44   enum BadQualityMetric { kDropFrame, kReportLowQP };
     45 
     46   QualityScalerTest() {
     47     input_frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, kHalfWidth,
     48                                   kHalfWidth);
     49     qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false);
     50     qs_.ReportFramerate(kFramerate);
     51     qs_.OnEncodeFrame(input_frame_);
     52   }
     53 
     54   bool TriggerScale(ScaleDirection scale_direction) {
     55     qs_.OnEncodeFrame(input_frame_);
     56     int initial_width = qs_.GetScaledResolution().width;
     57     for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
     58       switch (scale_direction) {
     59         case kScaleUp:
     60           qs_.ReportQP(kLowQp);
     61           break;
     62         case kScaleDown:
     63           qs_.ReportDroppedFrame();
     64           break;
     65         case kKeepScaleAtHighQp:
     66           qs_.ReportQP(kHighQp);
     67           break;
     68         case kScaleDownAboveHighQp:
     69           qs_.ReportQP(kHighQp + 1);
     70           break;
     71       }
     72       qs_.OnEncodeFrame(input_frame_);
     73       if (qs_.GetScaledResolution().width != initial_width)
     74         return true;
     75     }
     76 
     77     return false;
     78   }
     79 
     80   void ExpectOriginalFrame() {
     81     EXPECT_EQ(&input_frame_, &qs_.GetScaledFrame(input_frame_))
     82         << "Using scaled frame instead of original input.";
     83   }
     84 
     85   void ExpectScaleUsingReportedResolution() {
     86     qs_.OnEncodeFrame(input_frame_);
     87     QualityScaler::Resolution res = qs_.GetScaledResolution();
     88     const VideoFrame& scaled_frame = qs_.GetScaledFrame(input_frame_);
     89     EXPECT_EQ(res.width, scaled_frame.width());
     90     EXPECT_EQ(res.height, scaled_frame.height());
     91   }
     92 
     93   void ContinuouslyDownscalesByHalfDimensionsAndBackUp();
     94 
     95   void DoesNotDownscaleFrameDimensions(int width, int height);
     96 
     97   Resolution TriggerResolutionChange(BadQualityMetric dropframe_lowqp,
     98                                      int num_second,
     99                                      int initial_framerate);
    100 
    101   void VerifyQualityAdaptation(int initial_framerate,
    102                                int seconds,
    103                                bool expect_spatial_resize,
    104                                bool expect_framerate_reduction);
    105 
    106   void DownscaleEndsAt(int input_width,
    107                        int input_height,
    108                        int end_width,
    109                        int end_height);
    110 
    111   QualityScaler qs_;
    112   VideoFrame input_frame_;
    113 };
    114 
    115 TEST_F(QualityScalerTest, UsesOriginalFrameInitially) {
    116   ExpectOriginalFrame();
    117 }
    118 
    119 TEST_F(QualityScalerTest, ReportsOriginalResolutionInitially) {
    120   qs_.OnEncodeFrame(input_frame_);
    121   QualityScaler::Resolution res = qs_.GetScaledResolution();
    122   EXPECT_EQ(input_frame_.width(), res.width);
    123   EXPECT_EQ(input_frame_.height(), res.height);
    124 }
    125 
    126 TEST_F(QualityScalerTest, DownscalesAfterContinuousFramedrop) {
    127   EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within " << kNumSeconds
    128                                         << " seconds.";
    129   QualityScaler::Resolution res = qs_.GetScaledResolution();
    130   EXPECT_LT(res.width, input_frame_.width());
    131   EXPECT_LT(res.height, input_frame_.height());
    132 }
    133 
    134 TEST_F(QualityScalerTest, KeepsScaleAtHighQp) {
    135   EXPECT_FALSE(TriggerScale(kKeepScaleAtHighQp))
    136       << "Downscale at high threshold which should keep scale.";
    137   QualityScaler::Resolution res = qs_.GetScaledResolution();
    138   EXPECT_EQ(res.width, input_frame_.width());
    139   EXPECT_EQ(res.height, input_frame_.height());
    140 }
    141 
    142 TEST_F(QualityScalerTest, DownscalesAboveHighQp) {
    143   EXPECT_TRUE(TriggerScale(kScaleDownAboveHighQp))
    144       << "No downscale within " << kNumSeconds << " seconds.";
    145   QualityScaler::Resolution res = qs_.GetScaledResolution();
    146   EXPECT_LT(res.width, input_frame_.width());
    147   EXPECT_LT(res.height, input_frame_.height());
    148 }
    149 
    150 TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
    151   for (int i = 0; i < kFramerate * kNumSeconds / 3; ++i) {
    152     qs_.ReportQP(kNormalQp);
    153     qs_.ReportDroppedFrame();
    154     qs_.ReportDroppedFrame();
    155     qs_.OnEncodeFrame(input_frame_);
    156     if (qs_.GetScaledResolution().width < input_frame_.width())
    157       return;
    158   }
    159 
    160   FAIL() << "No downscale within " << kNumSeconds << " seconds.";
    161 }
    162 
    163 TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) {
    164   for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
    165     qs_.ReportQP(kNormalQp);
    166     qs_.OnEncodeFrame(input_frame_);
    167     ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
    168         << "Unexpected scale on half framedrop.";
    169   }
    170 }
    171 
    172 TEST_F(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) {
    173   for (int i = 0; i < kFramerate * kNumSeconds / 2; ++i) {
    174     qs_.ReportQP(kNormalQp);
    175     qs_.OnEncodeFrame(input_frame_);
    176     ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
    177         << "Unexpected scale on half framedrop.";
    178 
    179     qs_.ReportDroppedFrame();
    180     qs_.OnEncodeFrame(input_frame_);
    181     ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
    182         << "Unexpected scale on half framedrop.";
    183   }
    184 }
    185 
    186 void QualityScalerTest::ContinuouslyDownscalesByHalfDimensionsAndBackUp() {
    187   const int initial_min_dimension = input_frame_.width() < input_frame_.height()
    188                                         ? input_frame_.width()
    189                                         : input_frame_.height();
    190   int min_dimension = initial_min_dimension;
    191   int current_shift = 0;
    192   // Drop all frames to force-trigger downscaling.
    193   while (min_dimension >= 2 * QualityScaler::kDefaultMinDownscaleDimension) {
    194     EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within "
    195                                           << kNumSeconds << " seconds.";
    196     qs_.OnEncodeFrame(input_frame_);
    197     QualityScaler::Resolution res = qs_.GetScaledResolution();
    198     min_dimension = res.width < res.height ? res.width : res.height;
    199     ++current_shift;
    200     ASSERT_EQ(input_frame_.width() >> current_shift, res.width);
    201     ASSERT_EQ(input_frame_.height() >> current_shift, res.height);
    202     ExpectScaleUsingReportedResolution();
    203   }
    204 
    205   // Make sure we can scale back with good-quality frames.
    206   while (min_dimension < initial_min_dimension) {
    207     EXPECT_TRUE(TriggerScale(kScaleUp)) << "No upscale within " << kNumSeconds
    208                                         << " seconds.";
    209     qs_.OnEncodeFrame(input_frame_);
    210     QualityScaler::Resolution res = qs_.GetScaledResolution();
    211     min_dimension = res.width < res.height ? res.width : res.height;
    212     --current_shift;
    213     ASSERT_EQ(input_frame_.width() >> current_shift, res.width);
    214     ASSERT_EQ(input_frame_.height() >> current_shift, res.height);
    215     ExpectScaleUsingReportedResolution();
    216   }
    217 
    218   // Verify we don't start upscaling after further low use.
    219   for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
    220     qs_.ReportQP(kLowQp);
    221     ExpectOriginalFrame();
    222   }
    223 }
    224 
    225 TEST_F(QualityScalerTest, ContinuouslyDownscalesByHalfDimensionsAndBackUp) {
    226   ContinuouslyDownscalesByHalfDimensionsAndBackUp();
    227 }
    228 
    229 TEST_F(QualityScalerTest,
    230        ContinuouslyDownscalesOddResolutionsByHalfDimensionsAndBackUp) {
    231   const int kOddWidth = 517;
    232   const int kHalfOddWidth = (kOddWidth + 1) / 2;
    233   const int kOddHeight = 1239;
    234   input_frame_.CreateEmptyFrame(kOddWidth, kOddHeight, kOddWidth, kHalfOddWidth,
    235                                 kHalfOddWidth);
    236   ContinuouslyDownscalesByHalfDimensionsAndBackUp();
    237 }
    238 
    239 void QualityScalerTest::DoesNotDownscaleFrameDimensions(int width, int height) {
    240   input_frame_.CreateEmptyFrame(width, height, width, (width + 1) / 2,
    241                                 (width + 1) / 2);
    242 
    243   for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
    244     qs_.ReportDroppedFrame();
    245     qs_.OnEncodeFrame(input_frame_);
    246     ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
    247         << "Unexpected scale of minimal-size frame.";
    248   }
    249 }
    250 
    251 TEST_F(QualityScalerTest, DoesNotDownscaleFrom1PxWidth) {
    252   DoesNotDownscaleFrameDimensions(1, kHeight);
    253 }
    254 
    255 TEST_F(QualityScalerTest, DoesNotDownscaleFrom1PxHeight) {
    256   DoesNotDownscaleFrameDimensions(kWidth, 1);
    257 }
    258 
    259 TEST_F(QualityScalerTest, DoesNotDownscaleFrom1Px) {
    260   DoesNotDownscaleFrameDimensions(1, 1);
    261 }
    262 
    263 QualityScalerTest::Resolution QualityScalerTest::TriggerResolutionChange(
    264     BadQualityMetric dropframe_lowqp,
    265     int num_second,
    266     int initial_framerate) {
    267   QualityScalerTest::Resolution res;
    268   res.framerate = initial_framerate;
    269   qs_.OnEncodeFrame(input_frame_);
    270   res.width = qs_.GetScaledResolution().width;
    271   res.height = qs_.GetScaledResolution().height;
    272   for (int i = 0; i < kFramerate * num_second; ++i) {
    273     switch (dropframe_lowqp) {
    274       case kReportLowQP:
    275         qs_.ReportQP(kLowQp);
    276         break;
    277       case kDropFrame:
    278         qs_.ReportDroppedFrame();
    279         break;
    280     }
    281     qs_.OnEncodeFrame(input_frame_);
    282     // Simulate the case when SetRates is called right after reducing
    283     // framerate.
    284     qs_.ReportFramerate(initial_framerate);
    285     res.framerate = qs_.GetTargetFramerate();
    286     if (res.framerate != -1)
    287       qs_.ReportFramerate(res.framerate);
    288     res.width = qs_.GetScaledResolution().width;
    289     res.height = qs_.GetScaledResolution().height;
    290   }
    291   return res;
    292 }
    293 
    294 void QualityScalerTest::VerifyQualityAdaptation(
    295     int initial_framerate,
    296     int seconds,
    297     bool expect_spatial_resize,
    298     bool expect_framerate_reduction) {
    299   const int kDisabledBadQpThreshold = kMaxQp + 1;
    300   qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator,
    301            kDisabledBadQpThreshold, true);
    302   qs_.OnEncodeFrame(input_frame_);
    303   int init_width = qs_.GetScaledResolution().width;
    304   int init_height = qs_.GetScaledResolution().height;
    305 
    306   // Test reducing framerate by dropping frame continuously.
    307   QualityScalerTest::Resolution res =
    308       TriggerResolutionChange(kDropFrame, seconds, initial_framerate);
    309 
    310   if (expect_framerate_reduction) {
    311     EXPECT_LT(res.framerate, initial_framerate);
    312   } else {
    313     // No framerate reduction, video decimator should be disabled.
    314     EXPECT_EQ(-1, res.framerate);
    315   }
    316 
    317   if (expect_spatial_resize) {
    318     EXPECT_LT(res.width, init_width);
    319     EXPECT_LT(res.height, init_height);
    320   } else {
    321     EXPECT_EQ(init_width, res.width);
    322     EXPECT_EQ(init_height, res.height);
    323   }
    324 
    325   // The "seconds * 1.5" is to ensure spatial resolution to recover.
    326   // For example, in 10 seconds test, framerate reduction happens in the first
    327   // 5 seconds from 30fps to 15fps and causes the buffer size to be half of the
    328   // original one. Then it will take only 75 samples to downscale (twice in 150
    329   // samples). So to recover the resolution changes, we need more than 10
    330   // seconds (i.e, seconds * 1.5). This is because the framerate increases
    331   // before spatial size recovers, so it will take 150 samples to recover
    332   // spatial size (300 for twice).
    333   res = TriggerResolutionChange(kReportLowQP, seconds * 1.5, initial_framerate);
    334   EXPECT_EQ(-1, res.framerate);
    335   EXPECT_EQ(init_width, res.width);
    336   EXPECT_EQ(init_height, res.height);
    337 }
    338 
    339 // In 5 seconds test, only framerate adjusting should happen.
    340 TEST_F(QualityScalerTest, ChangeFramerateOnly) {
    341   VerifyQualityAdaptation(kFramerate, 5, false, true);
    342 }
    343 
    344 // In 10 seconds test, framerate adjusting and scaling are both
    345 // triggered, it shows that scaling would happen after framerate
    346 // adjusting.
    347 TEST_F(QualityScalerTest, ChangeFramerateAndSpatialSize) {
    348   VerifyQualityAdaptation(kFramerate, 10, true, true);
    349 }
    350 
    351 // When starting from a low framerate, only spatial size will be changed.
    352 TEST_F(QualityScalerTest, ChangeSpatialSizeOnly) {
    353   qs_.ReportFramerate(kFramerate >> 1);
    354   VerifyQualityAdaptation(kFramerate >> 1, 10, true, false);
    355 }
    356 
    357 TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsWidth) {
    358   DoesNotDownscaleFrameDimensions(
    359       2 * QualityScaler::kDefaultMinDownscaleDimension - 1, 1000);
    360 }
    361 
    362 TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsHeight) {
    363   DoesNotDownscaleFrameDimensions(
    364       1000, 2 * QualityScaler::kDefaultMinDownscaleDimension - 1);
    365 }
    366 
    367 void QualityScalerTest::DownscaleEndsAt(int input_width,
    368                                         int input_height,
    369                                         int end_width,
    370                                         int end_height) {
    371   // Create a frame with 2x expected end width/height to verify that we can
    372   // scale down to expected end width/height.
    373   input_frame_.CreateEmptyFrame(input_width, input_height, input_width,
    374                                 (input_width + 1) / 2, (input_width + 1) / 2);
    375 
    376   int last_width = input_width;
    377   int last_height = input_height;
    378   // Drop all frames to force-trigger downscaling.
    379   while (true) {
    380     TriggerScale(kScaleDown);
    381     QualityScaler::Resolution res = qs_.GetScaledResolution();
    382     if (last_width == res.width) {
    383       EXPECT_EQ(last_height, res.height);
    384       EXPECT_EQ(end_width, res.width);
    385       EXPECT_EQ(end_height, res.height);
    386       break;
    387     }
    388     last_width = res.width;
    389     last_height = res.height;
    390   }
    391 }
    392 
    393 TEST_F(QualityScalerTest, DefaultDownscalesTo160x90) {
    394   DownscaleEndsAt(320, 180, 160, 90);
    395 }
    396 
    397 TEST_F(QualityScalerTest, DefaultDownscalesTo90x160) {
    398   DownscaleEndsAt(180, 320, 90, 160);
    399 }
    400 
    401 TEST_F(QualityScalerTest, DefaultDownscalesFrom1280x720To160x90) {
    402   DownscaleEndsAt(1280, 720, 160, 90);
    403 }
    404 
    405 TEST_F(QualityScalerTest, DefaultDoesntDownscaleBelow160x90) {
    406   DownscaleEndsAt(320 - 1, 180 - 1, 320 - 1, 180 - 1);
    407 }
    408 
    409 TEST_F(QualityScalerTest, DefaultDoesntDownscaleBelow90x160) {
    410   DownscaleEndsAt(180 - 1, 320 - 1, 180 - 1, 320 - 1);
    411 }
    412 
    413 TEST_F(QualityScalerTest, RespectsMinResolutionWidth) {
    414   // Should end at 200x100, as width can't go lower.
    415   qs_.SetMinResolution(200, 10);
    416   DownscaleEndsAt(1600, 800, 200, 100);
    417 }
    418 
    419 TEST_F(QualityScalerTest, RespectsMinResolutionHeight) {
    420   // Should end at 100x200, as height can't go lower.
    421   qs_.SetMinResolution(10, 200);
    422   DownscaleEndsAt(800, 1600, 100, 200);
    423 }
    424 
    425 }  // namespace webrtc
    426