Home | History | Annotate | Download | only in video_coding 
      1 /*
      2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include <vector>
     12 
     13 #include "testing/gtest/include/gtest/gtest.h"
     14 #include "webrtc/base/scoped_ptr.h"
     15 #include "webrtc/common.h"
     16 #include "webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h"
     17 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
     18 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
     19 #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
     20 #include "webrtc/modules/video_coding/include/mock/mock_vcm_callbacks.h"
     21 #include "webrtc/modules/video_coding/include/video_coding.h"
     22 #include "webrtc/modules/video_coding/video_coding_impl.h"
     23 #include "webrtc/modules/video_coding/test/test_util.h"
     24 #include "webrtc/system_wrappers/include/clock.h"
     25 #include "webrtc/test/frame_generator.h"
     26 #include "webrtc/test/testsupport/fileutils.h"
     27 
     28 using ::testing::_;
     29 using ::testing::AllOf;
     30 using ::testing::ElementsAre;
     31 using ::testing::ElementsAreArray;
     32 using ::testing::Field;
     33 using ::testing::NiceMock;
     34 using ::testing::Pointee;
     35 using ::testing::Return;
     36 using ::testing::FloatEq;
     37 using std::vector;
     38 using webrtc::test::FrameGenerator;
     39 
     40 namespace webrtc {
     41 namespace vcm {
     42 namespace {
     43 enum { kMaxNumberOfTemporalLayers = 3 };
     44 
     45 struct Vp8StreamInfo {
     46   float framerate_fps[kMaxNumberOfTemporalLayers];
     47   int bitrate_kbps[kMaxNumberOfTemporalLayers];
     48 };
     49 
     50 MATCHER_P(MatchesVp8StreamInfo, expected, "") {
     51   bool res = true;
     52   for (int tl = 0; tl < kMaxNumberOfTemporalLayers; ++tl) {
     53     if (fabs(expected.framerate_fps[tl] - arg.framerate_fps[tl]) > 0.5) {
     54       *result_listener << " framerate_fps[" << tl
     55                        << "] = " << arg.framerate_fps[tl] << " (expected "
     56                        << expected.framerate_fps[tl] << ") ";
     57       res = false;
     58     }
     59     if (abs(expected.bitrate_kbps[tl] - arg.bitrate_kbps[tl]) > 10) {
     60       *result_listener << " bitrate_kbps[" << tl
     61                        << "] = " << arg.bitrate_kbps[tl] << " (expected "
     62                        << expected.bitrate_kbps[tl] << ") ";
     63       res = false;
     64     }
     65   }
     66   return res;
     67 }
     68 
     69 class EmptyFrameGenerator : public FrameGenerator {
     70  public:
     71   EmptyFrameGenerator(int width, int height) : width_(width), height_(height) {}
     72   VideoFrame* NextFrame() override {
     73     frame_.reset(new VideoFrame());
     74     frame_->CreateEmptyFrame(width_, height_, width_, (width_ + 1) / 2,
     75                              (width_ + 1) / 2);
     76     return frame_.get();
     77   }
     78 
     79  private:
     80   const int width_;
     81   const int height_;
     82   rtc::scoped_ptr<VideoFrame> frame_;
     83 };
     84 
     85 class PacketizationCallback : public VCMPacketizationCallback {
     86  public:
     87   explicit PacketizationCallback(Clock* clock)
     88       : clock_(clock), start_time_ms_(clock_->TimeInMilliseconds()) {}
     89 
     90   virtual ~PacketizationCallback() {}
     91 
     92   int32_t SendData(uint8_t payload_type,
     93                    const EncodedImage& encoded_image,
     94                    const RTPFragmentationHeader& fragmentation_header,
     95                    const RTPVideoHeader* rtp_video_header) override {
     96     assert(rtp_video_header);
     97     frame_data_.push_back(FrameData(encoded_image._length, *rtp_video_header));
     98     return 0;
     99   }
    100 
    101   void Reset() {
    102     frame_data_.clear();
    103     start_time_ms_ = clock_->TimeInMilliseconds();
    104   }
    105 
    106   float FramerateFpsWithinTemporalLayer(int temporal_layer) {
    107     return CountFramesWithinTemporalLayer(temporal_layer) *
    108            (1000.0 / interval_ms());
    109   }
    110 
    111   float BitrateKbpsWithinTemporalLayer(int temporal_layer) {
    112     return SumPayloadBytesWithinTemporalLayer(temporal_layer) * 8.0 /
    113            interval_ms();
    114   }
    115 
    116   Vp8StreamInfo CalculateVp8StreamInfo() {
    117     Vp8StreamInfo info;
    118     for (int tl = 0; tl < 3; ++tl) {
    119       info.framerate_fps[tl] = FramerateFpsWithinTemporalLayer(tl);
    120       info.bitrate_kbps[tl] = BitrateKbpsWithinTemporalLayer(tl);
    121     }
    122     return info;
    123   }
    124 
    125  private:
    126   struct FrameData {
    127     FrameData() {}
    128 
    129     FrameData(size_t payload_size, const RTPVideoHeader& rtp_video_header)
    130         : payload_size(payload_size), rtp_video_header(rtp_video_header) {}
    131 
    132     size_t payload_size;
    133     RTPVideoHeader rtp_video_header;
    134   };
    135 
    136   int64_t interval_ms() {
    137     int64_t diff = (clock_->TimeInMilliseconds() - start_time_ms_);
    138     EXPECT_GT(diff, 0);
    139     return diff;
    140   }
    141 
    142   int CountFramesWithinTemporalLayer(int temporal_layer) {
    143     int frames = 0;
    144     for (size_t i = 0; i < frame_data_.size(); ++i) {
    145       EXPECT_EQ(kRtpVideoVp8, frame_data_[i].rtp_video_header.codec);
    146       const uint8_t temporal_idx =
    147           frame_data_[i].rtp_video_header.codecHeader.VP8.temporalIdx;
    148       if (temporal_idx <= temporal_layer || temporal_idx == kNoTemporalIdx)
    149         frames++;
    150     }
    151     return frames;
    152   }
    153 
    154   size_t SumPayloadBytesWithinTemporalLayer(int temporal_layer) {
    155     size_t payload_size = 0;
    156     for (size_t i = 0; i < frame_data_.size(); ++i) {
    157       EXPECT_EQ(kRtpVideoVp8, frame_data_[i].rtp_video_header.codec);
    158       const uint8_t temporal_idx =
    159           frame_data_[i].rtp_video_header.codecHeader.VP8.temporalIdx;
    160       if (temporal_idx <= temporal_layer || temporal_idx == kNoTemporalIdx)
    161         payload_size += frame_data_[i].payload_size;
    162     }
    163     return payload_size;
    164   }
    165 
    166   Clock* clock_;
    167   int64_t start_time_ms_;
    168   vector<FrameData> frame_data_;
    169 };
    170 
    171 class TestVideoSender : public ::testing::Test {
    172  protected:
    173   // Note: simulated clock starts at 1 seconds, since parts of webrtc use 0 as
    174   // a special case (e.g. frame rate in media optimization).
    175   TestVideoSender() : clock_(1000), packetization_callback_(&clock_) {}
    176 
    177   void SetUp() override {
    178     sender_.reset(
    179         new VideoSender(&clock_, &post_encode_callback_, nullptr, nullptr));
    180     EXPECT_EQ(0, sender_->RegisterTransportCallback(&packetization_callback_));
    181   }
    182 
    183   void AddFrame() {
    184     assert(generator_.get());
    185     sender_->AddVideoFrame(*generator_->NextFrame(), NULL, NULL);
    186   }
    187 
    188   SimulatedClock clock_;
    189   PacketizationCallback packetization_callback_;
    190   MockEncodedImageCallback post_encode_callback_;
    191   // Used by subclassing tests, need to outlive sender_.
    192   rtc::scoped_ptr<VideoEncoder> encoder_;
    193   rtc::scoped_ptr<VideoSender> sender_;
    194   rtc::scoped_ptr<FrameGenerator> generator_;
    195 };
    196 
    197 class TestVideoSenderWithMockEncoder : public TestVideoSender {
    198  protected:
    199   static const int kDefaultWidth = 1280;
    200   static const int kDefaultHeight = 720;
    201   static const int kNumberOfStreams = 3;
    202   static const int kNumberOfLayers = 3;
    203   static const int kUnusedPayloadType = 10;
    204 
    205   void SetUp() override {
    206     TestVideoSender::SetUp();
    207     sender_->RegisterExternalEncoder(&encoder_, kUnusedPayloadType, false);
    208     VideoCodingModule::Codec(kVideoCodecVP8, &settings_);
    209     settings_.numberOfSimulcastStreams = kNumberOfStreams;
    210     ConfigureStream(kDefaultWidth / 4, kDefaultHeight / 4, 100,
    211                     &settings_.simulcastStream[0]);
    212     ConfigureStream(kDefaultWidth / 2, kDefaultHeight / 2, 500,
    213                     &settings_.simulcastStream[1]);
    214     ConfigureStream(kDefaultWidth, kDefaultHeight, 1200,
    215                     &settings_.simulcastStream[2]);
    216     settings_.plType = kUnusedPayloadType;  // Use the mocked encoder.
    217     generator_.reset(
    218         new EmptyFrameGenerator(settings_.width, settings_.height));
    219     EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200));
    220   }
    221 
    222   void TearDown() override { sender_.reset(); }
    223 
    224   void ExpectIntraRequest(int stream) {
    225     if (stream == -1) {
    226       // No intra request expected.
    227       EXPECT_CALL(
    228           encoder_,
    229           Encode(_, _, Pointee(ElementsAre(kVideoFrameDelta, kVideoFrameDelta,
    230                                            kVideoFrameDelta))))
    231           .Times(1)
    232           .WillRepeatedly(Return(0));
    233       return;
    234     }
    235     assert(stream >= 0);
    236     assert(stream < kNumberOfStreams);
    237     std::vector<FrameType> frame_types(kNumberOfStreams, kVideoFrameDelta);
    238     frame_types[stream] = kVideoFrameKey;
    239     EXPECT_CALL(encoder_,
    240                 Encode(_, _, Pointee(ElementsAreArray(&frame_types[0],
    241                                                       frame_types.size()))))
    242         .Times(1)
    243         .WillRepeatedly(Return(0));
    244   }
    245 
    246   static void ConfigureStream(int width,
    247                               int height,
    248                               int max_bitrate,
    249                               SimulcastStream* stream) {
    250     assert(stream);
    251     stream->width = width;
    252     stream->height = height;
    253     stream->maxBitrate = max_bitrate;
    254     stream->numberOfTemporalLayers = kNumberOfLayers;
    255     stream->qpMax = 45;
    256   }
    257 
    258   VideoCodec settings_;
    259   NiceMock<MockVideoEncoder> encoder_;
    260 };
    261 
    262 TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequests) {
    263   EXPECT_EQ(0, sender_->IntraFrameRequest(0));
    264   ExpectIntraRequest(0);
    265   AddFrame();
    266   ExpectIntraRequest(-1);
    267   AddFrame();
    268 
    269   EXPECT_EQ(0, sender_->IntraFrameRequest(1));
    270   ExpectIntraRequest(1);
    271   AddFrame();
    272   ExpectIntraRequest(-1);
    273   AddFrame();
    274 
    275   EXPECT_EQ(0, sender_->IntraFrameRequest(2));
    276   ExpectIntraRequest(2);
    277   AddFrame();
    278   ExpectIntraRequest(-1);
    279   AddFrame();
    280 
    281   EXPECT_EQ(-1, sender_->IntraFrameRequest(3));
    282   ExpectIntraRequest(-1);
    283   AddFrame();
    284 
    285   EXPECT_EQ(-1, sender_->IntraFrameRequest(-1));
    286   ExpectIntraRequest(-1);
    287   AddFrame();
    288 }
    289 
    290 TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequestsInternalCapture) {
    291   // De-register current external encoder.
    292   sender_->RegisterExternalEncoder(nullptr, kUnusedPayloadType, false);
    293   // Register encoder with internal capture.
    294   sender_->RegisterExternalEncoder(&encoder_, kUnusedPayloadType, true);
    295   EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200));
    296   ExpectIntraRequest(0);
    297   EXPECT_EQ(0, sender_->IntraFrameRequest(0));
    298   ExpectIntraRequest(1);
    299   EXPECT_EQ(0, sender_->IntraFrameRequest(1));
    300   ExpectIntraRequest(2);
    301   EXPECT_EQ(0, sender_->IntraFrameRequest(2));
    302   // No requests expected since these indices are out of bounds.
    303   EXPECT_EQ(-1, sender_->IntraFrameRequest(3));
    304   EXPECT_EQ(-1, sender_->IntraFrameRequest(-1));
    305 }
    306 
    307 TEST_F(TestVideoSenderWithMockEncoder, EncoderFramerateUpdatedViaProcess) {
    308   sender_->SetChannelParameters(settings_.startBitrate * 1000, 0, 200);
    309   const int64_t kRateStatsWindowMs = 2000;
    310   const uint32_t kInputFps = 20;
    311   int64_t start_time = clock_.TimeInMilliseconds();
    312   while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) {
    313     AddFrame();
    314     clock_.AdvanceTimeMilliseconds(1000 / kInputFps);
    315   }
    316   EXPECT_CALL(encoder_, SetRates(_, kInputFps)).Times(1).WillOnce(Return(0));
    317   sender_->Process();
    318   AddFrame();
    319 }
    320 
    321 TEST_F(TestVideoSenderWithMockEncoder,
    322        NoRedundantSetChannelParameterOrSetRatesCalls) {
    323   const uint8_t kLossRate = 4;
    324   const uint8_t kRtt = 200;
    325   const int64_t kRateStatsWindowMs = 2000;
    326   const uint32_t kInputFps = 20;
    327   int64_t start_time = clock_.TimeInMilliseconds();
    328   // Expect initial call to SetChannelParameters. Rates are initialized through
    329   // InitEncode and expects no additional call before the framerate (or bitrate)
    330   // updates.
    331   EXPECT_CALL(encoder_, SetChannelParameters(kLossRate, kRtt))
    332       .Times(1)
    333       .WillOnce(Return(0));
    334   sender_->SetChannelParameters(settings_.startBitrate * 1000, kLossRate, kRtt);
    335   while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) {
    336     AddFrame();
    337     clock_.AdvanceTimeMilliseconds(1000 / kInputFps);
    338   }
    339   // After process, input framerate should be updated but not ChannelParameters
    340   // as they are the same as before.
    341   EXPECT_CALL(encoder_, SetRates(_, kInputFps)).Times(1).WillOnce(Return(0));
    342   sender_->Process();
    343   AddFrame();
    344   // Call to SetChannelParameters with changed bitrate should call encoder
    345   // SetRates but not encoder SetChannelParameters (that are unchanged).
    346   EXPECT_CALL(encoder_, SetRates(2 * settings_.startBitrate, kInputFps))
    347       .Times(1)
    348       .WillOnce(Return(0));
    349   sender_->SetChannelParameters(2 * settings_.startBitrate * 1000, kLossRate,
    350                                 kRtt);
    351   AddFrame();
    352 }
    353 
    354 class TestVideoSenderWithVp8 : public TestVideoSender {
    355  public:
    356   TestVideoSenderWithVp8()
    357       : codec_bitrate_kbps_(300), available_bitrate_kbps_(1000) {}
    358 
    359   void SetUp() override {
    360     TestVideoSender::SetUp();
    361 
    362     const char* input_video = "foreman_cif";
    363     const int width = 352;
    364     const int height = 288;
    365     generator_.reset(FrameGenerator::CreateFromYuvFile(
    366         std::vector<std::string>(1, test::ResourcePath(input_video, "yuv")),
    367         width, height, 1));
    368 
    369     codec_ = MakeVp8VideoCodec(width, height, 3);
    370     codec_.minBitrate = 10;
    371     codec_.startBitrate = codec_bitrate_kbps_;
    372     codec_.maxBitrate = codec_bitrate_kbps_;
    373     encoder_.reset(VP8Encoder::Create());
    374     sender_->RegisterExternalEncoder(encoder_.get(), codec_.plType, false);
    375     EXPECT_EQ(0, sender_->RegisterSendCodec(&codec_, 1, 1200));
    376   }
    377 
    378   static VideoCodec MakeVp8VideoCodec(int width,
    379                                       int height,
    380                                       int temporal_layers) {
    381     VideoCodec codec;
    382     VideoCodingModule::Codec(kVideoCodecVP8, &codec);
    383     codec.width = width;
    384     codec.height = height;
    385     codec.codecSpecific.VP8.numberOfTemporalLayers = temporal_layers;
    386     return codec;
    387   }
    388 
    389   void InsertFrames(float framerate, float seconds) {
    390     for (int i = 0; i < seconds * framerate; ++i) {
    391       clock_.AdvanceTimeMilliseconds(1000.0f / framerate);
    392       EXPECT_CALL(post_encode_callback_, Encoded(_, NULL, NULL))
    393           .WillOnce(Return(0));
    394       AddFrame();
    395       // SetChannelParameters needs to be called frequently to propagate
    396       // framerate from the media optimization into the encoder.
    397       // Note: SetChannelParameters fails if less than 2 frames are in the
    398       // buffer since it will fail to calculate the framerate.
    399       if (i != 0) {
    400         EXPECT_EQ(VCM_OK, sender_->SetChannelParameters(
    401                               available_bitrate_kbps_ * 1000, 0, 200));
    402       }
    403     }
    404   }
    405 
    406   Vp8StreamInfo SimulateWithFramerate(float framerate) {
    407     const float short_simulation_interval = 5.0;
    408     const float long_simulation_interval = 10.0;
    409     // It appears that this 5 seconds simulation is needed to allow
    410     // bitrate and framerate to stabilize.
    411     InsertFrames(framerate, short_simulation_interval);
    412     packetization_callback_.Reset();
    413 
    414     InsertFrames(framerate, long_simulation_interval);
    415     return packetization_callback_.CalculateVp8StreamInfo();
    416   }
    417 
    418  protected:
    419   VideoCodec codec_;
    420   int codec_bitrate_kbps_;
    421   int available_bitrate_kbps_;
    422 };
    423 
    424 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
    425 #define MAYBE_FixedTemporalLayersStrategy DISABLED_FixedTemporalLayersStrategy
    426 #else
    427 #define MAYBE_FixedTemporalLayersStrategy FixedTemporalLayersStrategy
    428 #endif
    429 TEST_F(TestVideoSenderWithVp8, MAYBE_FixedTemporalLayersStrategy) {
    430   const int low_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][0];
    431   const int mid_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][1];
    432   const int high_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][2];
    433   {
    434     Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}};
    435     EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected));
    436   }
    437   {
    438     Vp8StreamInfo expected = {{3.75, 7.5, 15.0}, {low_b, mid_b, high_b}};
    439     EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected));
    440   }
    441 }
    442 
    443 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
    444 #define MAYBE_RealTimeTemporalLayersStrategy \
    445   DISABLED_RealTimeTemporalLayersStrategy
    446 #else
    447 #define MAYBE_RealTimeTemporalLayersStrategy RealTimeTemporalLayersStrategy
    448 #endif
    449 TEST_F(TestVideoSenderWithVp8, MAYBE_RealTimeTemporalLayersStrategy) {
    450   Config extra_options;
    451   extra_options.Set<TemporalLayers::Factory>(
    452       new RealTimeTemporalLayersFactory());
    453   VideoCodec codec = MakeVp8VideoCodec(352, 288, 3);
    454   codec.extra_options = &extra_options;
    455   codec.minBitrate = 10;
    456   codec.startBitrate = codec_bitrate_kbps_;
    457   codec.maxBitrate = codec_bitrate_kbps_;
    458   EXPECT_EQ(0, sender_->RegisterSendCodec(&codec, 1, 1200));
    459 
    460   const int low_b = codec_bitrate_kbps_ * 0.4;
    461   const int mid_b = codec_bitrate_kbps_ * 0.6;
    462   const int high_b = codec_bitrate_kbps_;
    463 
    464   {
    465     Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}};
    466     EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected));
    467   }
    468   {
    469     Vp8StreamInfo expected = {{5.0, 10.0, 20.0}, {low_b, mid_b, high_b}};
    470     EXPECT_THAT(SimulateWithFramerate(20.0), MatchesVp8StreamInfo(expected));
    471   }
    472   {
    473     Vp8StreamInfo expected = {{7.5, 15.0, 15.0}, {mid_b, high_b, high_b}};
    474     EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected));
    475   }
    476   {
    477     Vp8StreamInfo expected = {{5.0, 10.0, 10.0}, {mid_b, high_b, high_b}};
    478     EXPECT_THAT(SimulateWithFramerate(10.0), MatchesVp8StreamInfo(expected));
    479   }
    480   {
    481     // TODO(andresp): Find out why this fails with framerate = 7.5
    482     Vp8StreamInfo expected = {{7.0, 7.0, 7.0}, {high_b, high_b, high_b}};
    483     EXPECT_THAT(SimulateWithFramerate(7.0), MatchesVp8StreamInfo(expected));
    484   }
    485 }
    486 }  // namespace
    487 }  // namespace vcm
    488 }  // namespace webrtc
    489