Home | History | Annotate | Download | only in video 
      1 /*
      2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 #include <stdio.h>
     11 
     12 #include <deque>
     13 #include <map>
     14 
     15 #include "testing/gtest/include/gtest/gtest.h"
     16 
     17 #include "webrtc/base/thread_annotations.h"
     18 #include "webrtc/call.h"
     19 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
     20 #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
     21 #include "webrtc/system_wrappers/interface/clock.h"
     22 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
     23 #include "webrtc/system_wrappers/interface/event_wrapper.h"
     24 #include "webrtc/system_wrappers/interface/scoped_ptr.h"
     25 #include "webrtc/system_wrappers/interface/sleep.h"
     26 #include "webrtc/test/call_test.h"
     27 #include "webrtc/test/direct_transport.h"
     28 #include "webrtc/test/encoder_settings.h"
     29 #include "webrtc/test/fake_encoder.h"
     30 #include "webrtc/test/frame_generator_capturer.h"
     31 #include "webrtc/test/statistics.h"
     32 #include "webrtc/test/testsupport/fileutils.h"
     33 #include "webrtc/typedefs.h"
     34 
     35 namespace webrtc {
     36 
     37 static const int kFullStackTestDurationSecs = 60;
     38 
     39 struct FullStackTestParams {
     40   const char* test_label;
     41   struct {
     42     const char* name;
     43     size_t width, height;
     44     int fps;
     45   } clip;
     46   int min_bitrate_bps;
     47   int target_bitrate_bps;
     48   int max_bitrate_bps;
     49   double avg_psnr_threshold;
     50   double avg_ssim_threshold;
     51   FakeNetworkPipe::Config link;
     52 };
     53 
     54 class FullStackTest : public test::CallTest {
     55  protected:
     56   void RunTest(const FullStackTestParams& params);
     57 };
     58 
     59 class VideoAnalyzer : public PacketReceiver,
     60                       public newapi::Transport,
     61                       public VideoRenderer,
     62                       public VideoSendStreamInput {
     63  public:
     64   VideoAnalyzer(VideoSendStreamInput* input,
     65                 Transport* transport,
     66                 const char* test_label,
     67                 double avg_psnr_threshold,
     68                 double avg_ssim_threshold,
     69                 int duration_frames)
     70       : input_(input),
     71         transport_(transport),
     72         receiver_(NULL),
     73         test_label_(test_label),
     74         frames_left_(duration_frames),
     75         dropped_frames_(0),
     76         last_render_time_(0),
     77         rtp_timestamp_delta_(0),
     78         crit_(CriticalSectionWrapper::CreateCriticalSection()),
     79         first_send_frame_(NULL),
     80         avg_psnr_threshold_(avg_psnr_threshold),
     81         avg_ssim_threshold_(avg_ssim_threshold),
     82         comparison_lock_(CriticalSectionWrapper::CreateCriticalSection()),
     83         comparison_thread_(ThreadWrapper::CreateThread(&FrameComparisonThread,
     84                                                        this)),
     85         done_(EventWrapper::Create()) {
     86     unsigned int id;
     87     EXPECT_TRUE(comparison_thread_->Start(id));
     88   }
     89 
     90   ~VideoAnalyzer() {
     91     EXPECT_TRUE(comparison_thread_->Stop());
     92 
     93     while (!frames_.empty()) {
     94       delete frames_.back();
     95       frames_.pop_back();
     96     }
     97     while (!frame_pool_.empty()) {
     98       delete frame_pool_.back();
     99       frame_pool_.pop_back();
    100     }
    101   }
    102 
    103   virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; }
    104 
    105   virtual DeliveryStatus DeliverPacket(const uint8_t* packet,
    106                                        size_t length) OVERRIDE {
    107     scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
    108     RTPHeader header;
    109     parser->Parse(packet, length, &header);
    110     {
    111       CriticalSectionScoped lock(crit_.get());
    112       recv_times_[header.timestamp - rtp_timestamp_delta_] =
    113           Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
    114     }
    115 
    116     return receiver_->DeliverPacket(packet, length);
    117   }
    118 
    119   virtual void SwapFrame(I420VideoFrame* video_frame) OVERRIDE {
    120     I420VideoFrame* copy = NULL;
    121     {
    122       CriticalSectionScoped lock(crit_.get());
    123       if (frame_pool_.size() > 0) {
    124         copy = frame_pool_.front();
    125         frame_pool_.pop_front();
    126       }
    127     }
    128     if (copy == NULL)
    129       copy = new I420VideoFrame();
    130 
    131     copy->CopyFrame(*video_frame);
    132     copy->set_timestamp(copy->render_time_ms() * 90);
    133 
    134     {
    135       CriticalSectionScoped lock(crit_.get());
    136       if (first_send_frame_ == NULL && rtp_timestamp_delta_ == 0)
    137         first_send_frame_ = copy;
    138 
    139       frames_.push_back(copy);
    140     }
    141 
    142     input_->SwapFrame(video_frame);
    143   }
    144 
    145   virtual bool SendRtp(const uint8_t* packet, size_t length) OVERRIDE {
    146     scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
    147     RTPHeader header;
    148     parser->Parse(packet, length, &header);
    149 
    150     {
    151       CriticalSectionScoped lock(crit_.get());
    152       if (rtp_timestamp_delta_ == 0) {
    153         rtp_timestamp_delta_ =
    154             header.timestamp - first_send_frame_->timestamp();
    155         first_send_frame_ = NULL;
    156       }
    157       send_times_[header.timestamp - rtp_timestamp_delta_] =
    158           Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
    159     }
    160 
    161     return transport_->SendRtp(packet, length);
    162   }
    163 
    164   virtual bool SendRtcp(const uint8_t* packet, size_t length) OVERRIDE {
    165     return transport_->SendRtcp(packet, length);
    166   }
    167 
    168   virtual void RenderFrame(const I420VideoFrame& video_frame,
    169                            int time_to_render_ms) OVERRIDE {
    170     int64_t render_time_ms =
    171         Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
    172     uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_;
    173 
    174     CriticalSectionScoped lock(crit_.get());
    175     while (frames_.front()->timestamp() < send_timestamp) {
    176       AddFrameComparison(
    177           frames_.front(), &last_rendered_frame_, true, render_time_ms);
    178       frame_pool_.push_back(frames_.front());
    179       frames_.pop_front();
    180     }
    181 
    182     I420VideoFrame* reference_frame = frames_.front();
    183     frames_.pop_front();
    184     assert(reference_frame != NULL);
    185     EXPECT_EQ(reference_frame->timestamp(), send_timestamp);
    186     assert(reference_frame->timestamp() == send_timestamp);
    187 
    188     AddFrameComparison(reference_frame, &video_frame, false, render_time_ms);
    189     frame_pool_.push_back(reference_frame);
    190 
    191     last_rendered_frame_.CopyFrame(video_frame);
    192   }
    193 
    194   void Wait() {
    195     EXPECT_EQ(kEventSignaled, done_->Wait(FullStackTest::kLongTimeoutMs));
    196   }
    197 
    198   VideoSendStreamInput* input_;
    199   Transport* transport_;
    200   PacketReceiver* receiver_;
    201 
    202  private:
    203   struct FrameComparison {
    204     FrameComparison(const I420VideoFrame* reference,
    205                     const I420VideoFrame* render,
    206                     bool dropped,
    207                     int64_t send_time_ms,
    208                     int64_t recv_time_ms,
    209                     int64_t render_time_ms)
    210         : dropped(dropped),
    211           send_time_ms(send_time_ms),
    212           recv_time_ms(recv_time_ms),
    213           render_time_ms(render_time_ms) {
    214       this->reference.CopyFrame(*reference);
    215       this->render.CopyFrame(*render);
    216     }
    217 
    218     FrameComparison(const FrameComparison& compare)
    219         : dropped(compare.dropped),
    220           send_time_ms(compare.send_time_ms),
    221           recv_time_ms(compare.recv_time_ms),
    222           render_time_ms(compare.render_time_ms) {
    223       this->reference.CopyFrame(compare.reference);
    224       this->render.CopyFrame(compare.render);
    225     }
    226 
    227     ~FrameComparison() {}
    228 
    229     I420VideoFrame reference;
    230     I420VideoFrame render;
    231     bool dropped;
    232     int64_t send_time_ms;
    233     int64_t recv_time_ms;
    234     int64_t render_time_ms;
    235   };
    236 
    237   void AddFrameComparison(const I420VideoFrame* reference,
    238                           const I420VideoFrame* render,
    239                           bool dropped,
    240                           int64_t render_time_ms)
    241       EXCLUSIVE_LOCKS_REQUIRED(crit_) {
    242     int64_t send_time_ms = send_times_[reference->timestamp()];
    243     send_times_.erase(reference->timestamp());
    244     int64_t recv_time_ms = recv_times_[reference->timestamp()];
    245     recv_times_.erase(reference->timestamp());
    246 
    247     CriticalSectionScoped crit(comparison_lock_.get());
    248     comparisons_.push_back(FrameComparison(reference,
    249                                            render,
    250                                            dropped,
    251                                            send_time_ms,
    252                                            recv_time_ms,
    253                                            render_time_ms));
    254   }
    255 
    256   static bool FrameComparisonThread(void* obj) {
    257     return static_cast<VideoAnalyzer*>(obj)->CompareFrames();
    258   }
    259 
    260   bool CompareFrames() {
    261     assert(frames_left_ > 0);
    262 
    263     I420VideoFrame reference;
    264     I420VideoFrame render;
    265     bool dropped;
    266     int64_t send_time_ms;
    267     int64_t recv_time_ms;
    268     int64_t render_time_ms;
    269 
    270     SleepMs(10);
    271 
    272     while (true) {
    273       {
    274         CriticalSectionScoped crit(comparison_lock_.get());
    275         if (comparisons_.empty())
    276           return true;
    277         reference.SwapFrame(&comparisons_.front().reference);
    278         render.SwapFrame(&comparisons_.front().render);
    279         dropped = comparisons_.front().dropped;
    280         send_time_ms = comparisons_.front().send_time_ms;
    281         recv_time_ms = comparisons_.front().recv_time_ms;
    282         render_time_ms = comparisons_.front().render_time_ms;
    283         comparisons_.pop_front();
    284       }
    285 
    286       PerformFrameComparison(&reference,
    287                              &render,
    288                              dropped,
    289                              send_time_ms,
    290                              recv_time_ms,
    291                              render_time_ms);
    292 
    293       if (--frames_left_ == 0) {
    294         PrintResult("psnr", psnr_, " dB");
    295         PrintResult("ssim", ssim_, "");
    296         PrintResult("sender_time", sender_time_, " ms");
    297         printf(
    298             "RESULT dropped_frames: %s = %d\n", test_label_, dropped_frames_);
    299         PrintResult("receiver_time", receiver_time_, " ms");
    300         PrintResult("total_delay_incl_network", end_to_end_, " ms");
    301         PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
    302         EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
    303         EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
    304         done_->Set();
    305 
    306         return false;
    307       }
    308     }
    309   }
    310 
    311   void PerformFrameComparison(const I420VideoFrame* reference,
    312                               const I420VideoFrame* render,
    313                               bool dropped,
    314                               int64_t send_time_ms,
    315                               int64_t recv_time_ms,
    316                               int64_t render_time_ms) {
    317     psnr_.AddSample(I420PSNR(reference, render));
    318     ssim_.AddSample(I420SSIM(reference, render));
    319     if (dropped) {
    320       ++dropped_frames_;
    321       return;
    322     }
    323     if (last_render_time_ != 0)
    324       rendered_delta_.AddSample(render_time_ms - last_render_time_);
    325     last_render_time_ = render_time_ms;
    326 
    327     int64_t input_time_ms = reference->render_time_ms();
    328     sender_time_.AddSample(send_time_ms - input_time_ms);
    329     receiver_time_.AddSample(render_time_ms - recv_time_ms);
    330     end_to_end_.AddSample(render_time_ms - input_time_ms);
    331   }
    332 
    333   void PrintResult(const char* result_type,
    334                    test::Statistics stats,
    335                    const char* unit) {
    336     printf("RESULT %s: %s = {%f, %f}%s\n",
    337            result_type,
    338            test_label_,
    339            stats.Mean(),
    340            stats.StandardDeviation(),
    341            unit);
    342   }
    343 
    344   const char* const test_label_;
    345   test::Statistics sender_time_;
    346   test::Statistics receiver_time_;
    347   test::Statistics psnr_;
    348   test::Statistics ssim_;
    349   test::Statistics end_to_end_;
    350   test::Statistics rendered_delta_;
    351   int frames_left_;
    352   int dropped_frames_;
    353   int64_t last_render_time_;
    354   uint32_t rtp_timestamp_delta_;
    355 
    356   const scoped_ptr<CriticalSectionWrapper> crit_;
    357   std::deque<I420VideoFrame*> frames_ GUARDED_BY(crit_);
    358   std::deque<I420VideoFrame*> frame_pool_ GUARDED_BY(crit_);
    359   I420VideoFrame last_rendered_frame_ GUARDED_BY(crit_);
    360   std::map<uint32_t, int64_t> send_times_ GUARDED_BY(crit_);
    361   std::map<uint32_t, int64_t> recv_times_ GUARDED_BY(crit_);
    362   I420VideoFrame* first_send_frame_ GUARDED_BY(crit_);
    363   double avg_psnr_threshold_ GUARDED_BY(crit_);
    364   double avg_ssim_threshold_ GUARDED_BY(crit_);
    365 
    366   const scoped_ptr<CriticalSectionWrapper> comparison_lock_;
    367   const scoped_ptr<ThreadWrapper> comparison_thread_;
    368   std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
    369   const scoped_ptr<EventWrapper> done_;
    370 };
    371 
    372 void FullStackTest::RunTest(const FullStackTestParams& params) {
    373   test::DirectTransport send_transport(params.link);
    374   test::DirectTransport recv_transport(params.link);
    375   VideoAnalyzer analyzer(NULL,
    376                          &send_transport,
    377                          params.test_label,
    378                          params.avg_psnr_threshold,
    379                          params.avg_ssim_threshold,
    380                          kFullStackTestDurationSecs * params.clip.fps);
    381 
    382   CreateCalls(Call::Config(&analyzer), Call::Config(&recv_transport));
    383 
    384   analyzer.SetReceiver(receiver_call_->Receiver());
    385   send_transport.SetReceiver(&analyzer);
    386   recv_transport.SetReceiver(sender_call_->Receiver());
    387 
    388   CreateSendConfig(1);
    389 
    390   scoped_ptr<VideoEncoder> encoder(
    391       VideoEncoder::Create(VideoEncoder::kVp8));
    392   send_config_.encoder_settings.encoder = encoder.get();
    393   send_config_.encoder_settings.payload_name = "VP8";
    394   send_config_.encoder_settings.payload_type = 124;
    395 
    396   VideoStream* stream = &encoder_config_.streams[0];
    397   stream->width = params.clip.width;
    398   stream->height = params.clip.height;
    399   stream->min_bitrate_bps = params.min_bitrate_bps;
    400   stream->target_bitrate_bps = params.target_bitrate_bps;
    401   stream->max_bitrate_bps = params.max_bitrate_bps;
    402   stream->max_framerate = params.clip.fps;
    403 
    404   CreateMatchingReceiveConfigs();
    405   receive_configs_[0].renderer = &analyzer;
    406 
    407   CreateStreams();
    408   analyzer.input_ = send_stream_->Input();
    409 
    410   frame_generator_capturer_.reset(
    411       test::FrameGeneratorCapturer::CreateFromYuvFile(
    412           &analyzer,
    413           test::ResourcePath(params.clip.name, "yuv").c_str(),
    414           params.clip.width,
    415           params.clip.height,
    416           params.clip.fps,
    417           Clock::GetRealTimeClock()));
    418 
    419   ASSERT_TRUE(frame_generator_capturer_.get() != NULL)
    420       << "Could not create capturer for " << params.clip.name
    421       << ".yuv. Is this resource file present?";
    422 
    423   Start();
    424 
    425   analyzer.Wait();
    426 
    427   send_transport.StopSending();
    428   recv_transport.StopSending();
    429 
    430   Stop();
    431 
    432   DestroyStreams();
    433 }
    434 
    435 TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) {
    436   FullStackTestParams paris_qcif = {"net_delay_0_0_plr_0",
    437                                     {"paris_qcif", 176, 144, 30},
    438                                     300000,
    439                                     300000,
    440                                     300000,
    441                                     36.0,
    442                                     0.96
    443                                    };
    444   RunTest(paris_qcif);
    445 }
    446 
    447 TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) {
    448   // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif.
    449   FullStackTestParams foreman_cif = {"foreman_cif_net_delay_0_0_plr_0",
    450                                      {"foreman_cif", 352, 288, 30},
    451                                      700000,
    452                                      700000,
    453                                      700000,
    454                                      0.0,
    455                                      0.0
    456                                     };
    457   RunTest(foreman_cif);
    458 }
    459 
    460 TEST_F(FullStackTest, ForemanCif500kbps) {
    461   FullStackTestParams foreman_cif = {"foreman_cif_500kbps",
    462                                      {"foreman_cif", 352, 288, 30},
    463                                      30000,
    464                                      500000,
    465                                      2000000,
    466                                      0.0,
    467                                      0.0
    468                                     };
    469   foreman_cif.link.queue_length_packets = 0;
    470   foreman_cif.link.queue_delay_ms = 0;
    471   foreman_cif.link.link_capacity_kbps = 500;
    472   RunTest(foreman_cif);
    473 }
    474 
    475 TEST_F(FullStackTest, ForemanCif500kbpsLimitedQueue) {
    476   FullStackTestParams foreman_cif = {"foreman_cif_500kbps_32pkts_queue",
    477                                      {"foreman_cif", 352, 288, 30},
    478                                      30000,
    479                                      500000,
    480                                      2000000,
    481                                      0.0,
    482                                      0.0
    483                                     };
    484   foreman_cif.link.queue_length_packets = 32;
    485   foreman_cif.link.queue_delay_ms = 0;
    486   foreman_cif.link.link_capacity_kbps = 500;
    487   RunTest(foreman_cif);
    488 }
    489 
    490 TEST_F(FullStackTest, ForemanCif500kbps100ms) {
    491   FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms",
    492                                      {"foreman_cif", 352, 288, 30},
    493                                      30000,
    494                                      500000,
    495                                      2000000,
    496                                      0.0,
    497                                      0.0
    498                                     };
    499   foreman_cif.link.queue_length_packets = 0;
    500   foreman_cif.link.queue_delay_ms = 100;
    501   foreman_cif.link.link_capacity_kbps = 500;
    502   RunTest(foreman_cif);
    503 }
    504 
    505 TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueue) {
    506   FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms_32pkts_queue",
    507                                      {"foreman_cif", 352, 288, 30},
    508                                      30000,
    509                                      500000,
    510                                      2000000,
    511                                      0.0,
    512                                      0.0
    513                                     };
    514   foreman_cif.link.queue_length_packets = 32;
    515   foreman_cif.link.queue_delay_ms = 100;
    516   foreman_cif.link.link_capacity_kbps = 500;
    517   RunTest(foreman_cif);
    518 }
    519 
    520 TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) {
    521   FullStackTestParams foreman_cif = {"foreman_cif_1000kbps_100ms_32pkts_queue",
    522                                      {"foreman_cif", 352, 288, 30},
    523                                      30000,
    524                                      2000000,
    525                                      2000000,
    526                                      0.0,
    527                                      0.0
    528                                     };
    529   foreman_cif.link.queue_length_packets = 32;
    530   foreman_cif.link.queue_delay_ms = 100;
    531   foreman_cif.link.link_capacity_kbps = 1000;
    532   RunTest(foreman_cif);
    533 }
    534 }  // namespace webrtc
    535