Home | History | Annotate | Download | only in test 
      1 /*
      2  *  Copyright (c) 2013 The WebM project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "third_party/googletest/src/include/gtest/gtest.h"
     12 #include "test/codec_factory.h"
     13 #include "test/encode_test_driver.h"
     14 #include "test/i420_video_source.h"
     15 #include "test/util.h"
     16 
     17 namespace {
     18 
     19 const int kMaxErrorFrames = 12;
     20 const int kMaxDroppableFrames = 12;
     21 
     22 class ErrorResilienceTestLarge : public ::libvpx_test::EncoderTest,
     23     public ::libvpx_test::CodecTestWithParam<libvpx_test::TestMode> {
     24  protected:
     25   ErrorResilienceTestLarge()
     26       : EncoderTest(GET_PARAM(0)),
     27         psnr_(0.0),
     28         nframes_(0),
     29         mismatch_psnr_(0.0),
     30         mismatch_nframes_(0),
     31         encoding_mode_(GET_PARAM(1)) {
     32     Reset();
     33   }
     34 
     35   virtual ~ErrorResilienceTestLarge() {}
     36 
     37   void Reset() {
     38     error_nframes_ = 0;
     39     droppable_nframes_ = 0;
     40   }
     41 
     42   virtual void SetUp() {
     43     InitializeConfig();
     44     SetMode(encoding_mode_);
     45   }
     46 
     47   virtual void BeginPassHook(unsigned int /*pass*/) {
     48     psnr_ = 0.0;
     49     nframes_ = 0;
     50     mismatch_psnr_ = 0.0;
     51     mismatch_nframes_ = 0;
     52   }
     53 
     54   virtual void PSNRPktHook(const vpx_codec_cx_pkt_t *pkt) {
     55     psnr_ += pkt->data.psnr.psnr[0];
     56     nframes_++;
     57   }
     58 
     59   virtual void PreEncodeFrameHook(libvpx_test::VideoSource *video) {
     60     frame_flags_ &= ~(VP8_EFLAG_NO_UPD_LAST |
     61                       VP8_EFLAG_NO_UPD_GF |
     62                       VP8_EFLAG_NO_UPD_ARF);
     63     if (droppable_nframes_ > 0 &&
     64         (cfg_.g_pass == VPX_RC_LAST_PASS || cfg_.g_pass == VPX_RC_ONE_PASS)) {
     65       for (unsigned int i = 0; i < droppable_nframes_; ++i) {
     66         if (droppable_frames_[i] == video->frame()) {
     67           std::cout << "             Encoding droppable frame: "
     68                     << droppable_frames_[i] << "\n";
     69           frame_flags_ |= (VP8_EFLAG_NO_UPD_LAST |
     70                            VP8_EFLAG_NO_UPD_GF |
     71                            VP8_EFLAG_NO_UPD_ARF);
     72           return;
     73         }
     74       }
     75     }
     76   }
     77 
     78   double GetAveragePsnr() const {
     79     if (nframes_)
     80       return psnr_ / nframes_;
     81     return 0.0;
     82   }
     83 
     84   double GetAverageMismatchPsnr() const {
     85     if (mismatch_nframes_)
     86       return mismatch_psnr_ / mismatch_nframes_;
     87     return 0.0;
     88   }
     89 
     90   virtual bool DoDecode() const {
     91     if (error_nframes_ > 0 &&
     92         (cfg_.g_pass == VPX_RC_LAST_PASS || cfg_.g_pass == VPX_RC_ONE_PASS)) {
     93       for (unsigned int i = 0; i < error_nframes_; ++i) {
     94         if (error_frames_[i] == nframes_ - 1) {
     95           std::cout << "             Skipping decoding frame: "
     96                     << error_frames_[i] << "\n";
     97           return 0;
     98         }
     99       }
    100     }
    101     return 1;
    102   }
    103 
    104   virtual void MismatchHook(const vpx_image_t *img1,
    105                             const vpx_image_t *img2) {
    106     double mismatch_psnr = compute_psnr(img1, img2);
    107     mismatch_psnr_ += mismatch_psnr;
    108     ++mismatch_nframes_;
    109     // std::cout << "Mismatch frame psnr: " << mismatch_psnr << "\n";
    110   }
    111 
    112   void SetErrorFrames(int num, unsigned int *list) {
    113     if (num > kMaxErrorFrames)
    114       num = kMaxErrorFrames;
    115     else if (num < 0)
    116       num = 0;
    117     error_nframes_ = num;
    118     for (unsigned int i = 0; i < error_nframes_; ++i)
    119       error_frames_[i] = list[i];
    120   }
    121 
    122   void SetDroppableFrames(int num, unsigned int *list) {
    123     if (num > kMaxDroppableFrames)
    124       num = kMaxDroppableFrames;
    125     else if (num < 0)
    126       num = 0;
    127     droppable_nframes_ = num;
    128     for (unsigned int i = 0; i < droppable_nframes_; ++i)
    129       droppable_frames_[i] = list[i];
    130   }
    131 
    132   unsigned int GetMismatchFrames() {
    133     return mismatch_nframes_;
    134   }
    135 
    136  private:
    137   double psnr_;
    138   unsigned int nframes_;
    139   unsigned int error_nframes_;
    140   unsigned int droppable_nframes_;
    141   double mismatch_psnr_;
    142   unsigned int mismatch_nframes_;
    143   unsigned int error_frames_[kMaxErrorFrames];
    144   unsigned int droppable_frames_[kMaxDroppableFrames];
    145   libvpx_test::TestMode encoding_mode_;
    146 };
    147 
    148 TEST_P(ErrorResilienceTestLarge, OnVersusOff) {
    149   const vpx_rational timebase = { 33333333, 1000000000 };
    150   cfg_.g_timebase = timebase;
    151   cfg_.rc_target_bitrate = 2000;
    152   cfg_.g_lag_in_frames = 10;
    153 
    154   init_flags_ = VPX_CODEC_USE_PSNR;
    155 
    156   libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
    157                                      timebase.den, timebase.num, 0, 30);
    158 
    159   // Error resilient mode OFF.
    160   cfg_.g_error_resilient = 0;
    161   ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
    162   const double psnr_resilience_off = GetAveragePsnr();
    163   EXPECT_GT(psnr_resilience_off, 25.0);
    164 
    165   // Error resilient mode ON.
    166   cfg_.g_error_resilient = 1;
    167   ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
    168   const double psnr_resilience_on = GetAveragePsnr();
    169   EXPECT_GT(psnr_resilience_on, 25.0);
    170 
    171   // Test that turning on error resilient mode hurts by 10% at most.
    172   if (psnr_resilience_off > 0.0) {
    173     const double psnr_ratio = psnr_resilience_on / psnr_resilience_off;
    174     EXPECT_GE(psnr_ratio, 0.9);
    175     EXPECT_LE(psnr_ratio, 1.1);
    176   }
    177 }
    178 
    179 // Check for successful decoding and no encoder/decoder mismatch
    180 // if we lose (i.e., drop before decoding) a set of droppable
    181 // frames (i.e., frames that don't update any reference buffers).
    182 // Check both isolated and consecutive loss.
    183 TEST_P(ErrorResilienceTestLarge, DropFramesWithoutRecovery) {
    184   const vpx_rational timebase = { 33333333, 1000000000 };
    185   cfg_.g_timebase = timebase;
    186   cfg_.rc_target_bitrate = 500;
    187   // FIXME(debargha): Fix this to work for any lag.
    188   // Currently this test only works for lag = 0
    189   cfg_.g_lag_in_frames = 0;
    190 
    191   init_flags_ = VPX_CODEC_USE_PSNR;
    192 
    193   libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
    194                                      timebase.den, timebase.num, 0, 40);
    195 
    196   // Error resilient mode ON.
    197   cfg_.g_error_resilient = 1;
    198   cfg_.kf_mode = VPX_KF_DISABLED;
    199 
    200   // Set an arbitrary set of error frames same as droppable frames.
    201   // In addition to isolated loss/drop, add a long consecutive series
    202   // (of size 9) of dropped frames.
    203   unsigned int num_droppable_frames = 11;
    204   unsigned int droppable_frame_list[] = {5, 16, 22, 23, 24, 25, 26, 27, 28,
    205                                          29, 30};
    206   SetDroppableFrames(num_droppable_frames, droppable_frame_list);
    207   SetErrorFrames(num_droppable_frames, droppable_frame_list);
    208   ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
    209   // Test that no mismatches have been found
    210   std::cout << "             Mismatch frames: "
    211             << GetMismatchFrames() << "\n";
    212   EXPECT_EQ(GetMismatchFrames(), (unsigned int) 0);
    213 
    214   // Reset previously set of error/droppable frames.
    215   Reset();
    216 
    217 #if 0
    218   // TODO(jkoleszar): This test is disabled for the time being as too
    219   // sensitive. It's not clear how to set a reasonable threshold for
    220   // this behavior.
    221 
    222   // Now set an arbitrary set of error frames that are non-droppable
    223   unsigned int num_error_frames = 3;
    224   unsigned int error_frame_list[] = {3, 10, 20};
    225   SetErrorFrames(num_error_frames, error_frame_list);
    226   ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
    227 
    228   // Test that dropping an arbitrary set of inter frames does not hurt too much
    229   // Note the Average Mismatch PSNR is the average of the PSNR between
    230   // decoded frame and encoder's version of the same frame for all frames
    231   // with mismatch.
    232   const double psnr_resilience_mismatch = GetAverageMismatchPsnr();
    233   std::cout << "             Mismatch PSNR: "
    234             << psnr_resilience_mismatch << "\n";
    235   EXPECT_GT(psnr_resilience_mismatch, 20.0);
    236 #endif
    237 }
    238 
    239 VP8_INSTANTIATE_TEST_CASE(ErrorResilienceTestLarge, ONE_PASS_TEST_MODES);
    240 VP9_INSTANTIATE_TEST_CASE(ErrorResilienceTestLarge, ONE_PASS_TEST_MODES);
    241 
    242 }  // namespace
    243