Home | History | Annotate | Download | only in video 
      1 /*
      2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 #include <stdio.h>
     11 
     12 #include <deque>
     13 #include <map>
     14 
     15 #include "testing/gtest/include/gtest/gtest.h"
     16 
     17 #include "webrtc/call.h"
     18 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
     19 #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
     20 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
     21 #include "webrtc/system_wrappers/interface/clock.h"
     22 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
     23 #include "webrtc/system_wrappers/interface/event_wrapper.h"
     24 #include "webrtc/system_wrappers/interface/scoped_ptr.h"
     25 #include "webrtc/system_wrappers/interface/sleep.h"
     26 #include "webrtc/system_wrappers/interface/thread_annotations.h"
     27 #include "webrtc/test/direct_transport.h"
     28 #include "webrtc/test/encoder_settings.h"
     29 #include "webrtc/test/fake_encoder.h"
     30 #include "webrtc/test/frame_generator_capturer.h"
     31 #include "webrtc/test/statistics.h"
     32 #include "webrtc/test/testsupport/fileutils.h"
     33 #include "webrtc/typedefs.h"
     34 
     35 namespace webrtc {
     36 
     37 static const uint32_t kSendSsrc = 0x654321;
     38 static const int kFullStackTestDurationSecs = 10;
     39 
     40 struct FullStackTestParams {
     41   const char* test_label;
     42   struct {
     43     const char* name;
     44     size_t width, height;
     45     int fps;
     46   } clip;
     47   unsigned int bitrate;
     48   double avg_psnr_threshold;
     49   double avg_ssim_threshold;
     50 };
     51 
     52 FullStackTestParams paris_qcif = {
     53     "net_delay_0_0_plr_0", {"paris_qcif", 176, 144, 30}, 300, 36.0, 0.96};
     54 
     55 // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif.
     56 FullStackTestParams foreman_cif = {
     57     "foreman_cif_net_delay_0_0_plr_0",
     58     {"foreman_cif", 352, 288, 30},
     59     700,
     60     0.0,
     61     0.0};
     62 
     63 class FullStackTest : public ::testing::TestWithParam<FullStackTestParams> {
     64  protected:
     65   std::map<uint32_t, bool> reserved_ssrcs_;
     66 };
     67 
     68 class VideoAnalyzer : public PacketReceiver,
     69                       public newapi::Transport,
     70                       public VideoRenderer,
     71                       public VideoSendStreamInput {
     72  public:
     73   VideoAnalyzer(VideoSendStreamInput* input,
     74                 Transport* transport,
     75                 const char* test_label,
     76                 double avg_psnr_threshold,
     77                 double avg_ssim_threshold,
     78                 int duration_frames)
     79       : input_(input),
     80         transport_(transport),
     81         receiver_(NULL),
     82         test_label_(test_label),
     83         frames_left_(duration_frames),
     84         dropped_frames_(0),
     85         last_render_time_(0),
     86         rtp_timestamp_delta_(0),
     87         crit_(CriticalSectionWrapper::CreateCriticalSection()),
     88         first_send_frame_(NULL),
     89         avg_psnr_threshold_(avg_psnr_threshold),
     90         avg_ssim_threshold_(avg_ssim_threshold),
     91         comparison_lock_(CriticalSectionWrapper::CreateCriticalSection()),
     92         comparison_thread_(ThreadWrapper::CreateThread(&FrameComparisonThread,
     93                                                        this)),
     94         done_(EventWrapper::Create()) {
     95     unsigned int id;
     96     EXPECT_TRUE(comparison_thread_->Start(id));
     97   }
     98 
     99   ~VideoAnalyzer() {
    100     EXPECT_TRUE(comparison_thread_->Stop());
    101 
    102     while (!frames_.empty()) {
    103       delete frames_.back();
    104       frames_.pop_back();
    105     }
    106     while (!frame_pool_.empty()) {
    107       delete frame_pool_.back();
    108       frame_pool_.pop_back();
    109     }
    110   }
    111 
    112   virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; }
    113 
    114   virtual DeliveryStatus DeliverPacket(const uint8_t* packet,
    115                                        size_t length) OVERRIDE {
    116     scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
    117     RTPHeader header;
    118     parser->Parse(packet, static_cast<int>(length), &header);
    119     {
    120       CriticalSectionScoped lock(crit_.get());
    121       recv_times_[header.timestamp - rtp_timestamp_delta_] =
    122           Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
    123     }
    124 
    125     return receiver_->DeliverPacket(packet, length);
    126   }
    127 
    128   virtual void SwapFrame(I420VideoFrame* video_frame) OVERRIDE {
    129     I420VideoFrame* copy = NULL;
    130     {
    131       CriticalSectionScoped lock(crit_.get());
    132       if (frame_pool_.size() > 0) {
    133         copy = frame_pool_.front();
    134         frame_pool_.pop_front();
    135       }
    136     }
    137     if (copy == NULL)
    138       copy = new I420VideoFrame();
    139 
    140     copy->CopyFrame(*video_frame);
    141     copy->set_timestamp(copy->render_time_ms() * 90);
    142 
    143     {
    144       CriticalSectionScoped lock(crit_.get());
    145       if (first_send_frame_ == NULL && rtp_timestamp_delta_ == 0)
    146         first_send_frame_ = copy;
    147 
    148       frames_.push_back(copy);
    149     }
    150 
    151     input_->SwapFrame(video_frame);
    152   }
    153 
    154   virtual bool SendRtp(const uint8_t* packet, size_t length) OVERRIDE {
    155     scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
    156     RTPHeader header;
    157     parser->Parse(packet, static_cast<int>(length), &header);
    158 
    159     {
    160       CriticalSectionScoped lock(crit_.get());
    161       if (rtp_timestamp_delta_ == 0) {
    162         rtp_timestamp_delta_ =
    163             header.timestamp - first_send_frame_->timestamp();
    164         first_send_frame_ = NULL;
    165       }
    166       send_times_[header.timestamp - rtp_timestamp_delta_] =
    167           Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
    168     }
    169 
    170     return transport_->SendRtp(packet, length);
    171   }
    172 
    173   virtual bool SendRtcp(const uint8_t* packet, size_t length) OVERRIDE {
    174     return transport_->SendRtcp(packet, length);
    175   }
    176 
    177   virtual void RenderFrame(const I420VideoFrame& video_frame,
    178                            int time_to_render_ms) OVERRIDE {
    179     int64_t render_time_ms =
    180         Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
    181     uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_;
    182 
    183     CriticalSectionScoped lock(crit_.get());
    184     while (frames_.front()->timestamp() < send_timestamp) {
    185       AddFrameComparison(
    186           frames_.front(), &last_rendered_frame_, true, render_time_ms);
    187       frame_pool_.push_back(frames_.front());
    188       frames_.pop_front();
    189     }
    190 
    191     I420VideoFrame* reference_frame = frames_.front();
    192     frames_.pop_front();
    193     assert(reference_frame != NULL);
    194     EXPECT_EQ(reference_frame->timestamp(), send_timestamp);
    195     assert(reference_frame->timestamp() == send_timestamp);
    196 
    197     AddFrameComparison(reference_frame, &video_frame, false, render_time_ms);
    198     frame_pool_.push_back(reference_frame);
    199 
    200     last_rendered_frame_.CopyFrame(video_frame);
    201   }
    202 
    203   void Wait() { done_->Wait(120 * 1000); }
    204 
    205   VideoSendStreamInput* input_;
    206   Transport* transport_;
    207   PacketReceiver* receiver_;
    208 
    209  private:
    210   struct FrameComparison {
    211     FrameComparison(const I420VideoFrame* reference,
    212                     const I420VideoFrame* render,
    213                     bool dropped,
    214                     int64_t send_time_ms,
    215                     int64_t recv_time_ms,
    216                     int64_t render_time_ms)
    217         : dropped(dropped),
    218           send_time_ms(send_time_ms),
    219           recv_time_ms(recv_time_ms),
    220           render_time_ms(render_time_ms) {
    221       this->reference.CopyFrame(*reference);
    222       this->render.CopyFrame(*render);
    223     }
    224 
    225     FrameComparison(const FrameComparison& compare)
    226         : dropped(compare.dropped),
    227           send_time_ms(compare.send_time_ms),
    228           recv_time_ms(compare.recv_time_ms),
    229           render_time_ms(compare.render_time_ms) {
    230       this->reference.CopyFrame(compare.reference);
    231       this->render.CopyFrame(compare.render);
    232     }
    233 
    234     ~FrameComparison() {}
    235 
    236     I420VideoFrame reference;
    237     I420VideoFrame render;
    238     bool dropped;
    239     int64_t send_time_ms;
    240     int64_t recv_time_ms;
    241     int64_t render_time_ms;
    242   };
    243 
    244   void AddFrameComparison(const I420VideoFrame* reference,
    245                           const I420VideoFrame* render,
    246                           bool dropped,
    247                           int64_t render_time_ms)
    248       EXCLUSIVE_LOCKS_REQUIRED(crit_) {
    249     int64_t send_time_ms = send_times_[reference->timestamp()];
    250     send_times_.erase(reference->timestamp());
    251     int64_t recv_time_ms = recv_times_[reference->timestamp()];
    252     recv_times_.erase(reference->timestamp());
    253 
    254     CriticalSectionScoped crit(comparison_lock_.get());
    255     comparisons_.push_back(FrameComparison(reference,
    256                                            render,
    257                                            dropped,
    258                                            send_time_ms,
    259                                            recv_time_ms,
    260                                            render_time_ms));
    261   }
    262 
    263   static bool FrameComparisonThread(void* obj) {
    264     return static_cast<VideoAnalyzer*>(obj)->CompareFrames();
    265   }
    266 
    267   bool CompareFrames() {
    268     assert(frames_left_ > 0);
    269 
    270     I420VideoFrame reference;
    271     I420VideoFrame render;
    272     bool dropped;
    273     int64_t send_time_ms;
    274     int64_t recv_time_ms;
    275     int64_t render_time_ms;
    276 
    277     SleepMs(10);
    278 
    279     while (true) {
    280       {
    281         CriticalSectionScoped crit(comparison_lock_.get());
    282         if (comparisons_.empty())
    283           return true;
    284         reference.SwapFrame(&comparisons_.front().reference);
    285         render.SwapFrame(&comparisons_.front().render);
    286         dropped = comparisons_.front().dropped;
    287         send_time_ms = comparisons_.front().send_time_ms;
    288         recv_time_ms = comparisons_.front().recv_time_ms;
    289         render_time_ms = comparisons_.front().render_time_ms;
    290         comparisons_.pop_front();
    291       }
    292 
    293       PerformFrameComparison(&reference,
    294                              &render,
    295                              dropped,
    296                              send_time_ms,
    297                              recv_time_ms,
    298                              render_time_ms);
    299 
    300       if (--frames_left_ == 0) {
    301         PrintResult("psnr", psnr_, " dB");
    302         PrintResult("ssim", ssim_, "");
    303         PrintResult("sender_time", sender_time_, " ms");
    304         printf(
    305             "RESULT dropped_frames: %s = %d\n", test_label_, dropped_frames_);
    306         PrintResult("receiver_time", receiver_time_, " ms");
    307         PrintResult("total_delay_incl_network", end_to_end_, " ms");
    308         PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
    309         EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
    310         EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
    311         done_->Set();
    312 
    313         return false;
    314       }
    315     }
    316   }
    317 
    318   void PerformFrameComparison(const I420VideoFrame* reference,
    319                               const I420VideoFrame* render,
    320                               bool dropped,
    321                               int64_t send_time_ms,
    322                               int64_t recv_time_ms,
    323                               int64_t render_time_ms) {
    324     psnr_.AddSample(I420PSNR(reference, render));
    325     ssim_.AddSample(I420SSIM(reference, render));
    326     if (dropped) {
    327       ++dropped_frames_;
    328       return;
    329     }
    330     if (last_render_time_ != 0)
    331       rendered_delta_.AddSample(render_time_ms - last_render_time_);
    332     last_render_time_ = render_time_ms;
    333 
    334     int64_t input_time_ms = reference->render_time_ms();
    335     sender_time_.AddSample(send_time_ms - input_time_ms);
    336     receiver_time_.AddSample(render_time_ms - recv_time_ms);
    337     end_to_end_.AddSample(render_time_ms - input_time_ms);
    338   }
    339 
    340   void PrintResult(const char* result_type,
    341                    test::Statistics stats,
    342                    const char* unit) {
    343     printf("RESULT %s: %s = {%f, %f}%s\n",
    344            result_type,
    345            test_label_,
    346            stats.Mean(),
    347            stats.StandardDeviation(),
    348            unit);
    349   }
    350 
    351   const char* const test_label_;
    352   test::Statistics sender_time_;
    353   test::Statistics receiver_time_;
    354   test::Statistics psnr_;
    355   test::Statistics ssim_;
    356   test::Statistics end_to_end_;
    357   test::Statistics rendered_delta_;
    358   int frames_left_;
    359   int dropped_frames_;
    360   int64_t last_render_time_;
    361   uint32_t rtp_timestamp_delta_;
    362 
    363   const scoped_ptr<CriticalSectionWrapper> crit_;
    364   std::deque<I420VideoFrame*> frames_ GUARDED_BY(crit_);
    365   std::deque<I420VideoFrame*> frame_pool_ GUARDED_BY(crit_);
    366   I420VideoFrame last_rendered_frame_ GUARDED_BY(crit_);
    367   std::map<uint32_t, int64_t> send_times_ GUARDED_BY(crit_);
    368   std::map<uint32_t, int64_t> recv_times_ GUARDED_BY(crit_);
    369   I420VideoFrame* first_send_frame_ GUARDED_BY(crit_);
    370   double avg_psnr_threshold_ GUARDED_BY(crit_);
    371   double avg_ssim_threshold_ GUARDED_BY(crit_);
    372 
    373   const scoped_ptr<CriticalSectionWrapper> comparison_lock_;
    374   const scoped_ptr<ThreadWrapper> comparison_thread_;
    375   std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
    376   const scoped_ptr<EventWrapper> done_;
    377 };
    378 
    379 TEST_P(FullStackTest, NoPacketLoss) {
    380   static const uint32_t kReceiverLocalSsrc = 0x123456;
    381   FullStackTestParams params = GetParam();
    382 
    383   test::DirectTransport transport;
    384   VideoAnalyzer analyzer(NULL,
    385                          &transport,
    386                          params.test_label,
    387                          params.avg_psnr_threshold,
    388                          params.avg_ssim_threshold,
    389                          kFullStackTestDurationSecs * params.clip.fps);
    390 
    391   Call::Config call_config(&analyzer);
    392 
    393   scoped_ptr<Call> call(Call::Create(call_config));
    394   analyzer.SetReceiver(call->Receiver());
    395   transport.SetReceiver(&analyzer);
    396 
    397   VideoSendStream::Config send_config = call->GetDefaultSendConfig();
    398   send_config.rtp.ssrcs.push_back(kSendSsrc);
    399 
    400   scoped_ptr<VP8Encoder> encoder(VP8Encoder::Create());
    401   send_config.encoder_settings.encoder = encoder.get();
    402   send_config.encoder_settings.payload_name = "VP8";
    403   send_config.encoder_settings.payload_type = 124;
    404   std::vector<VideoStream> video_streams = test::CreateVideoStreams(1);
    405   VideoStream* stream = &video_streams[0];
    406   stream->width = params.clip.width;
    407   stream->height = params.clip.height;
    408   stream->min_bitrate_bps = stream->target_bitrate_bps =
    409       stream->max_bitrate_bps = params.bitrate * 1000;
    410   stream->max_framerate = params.clip.fps;
    411 
    412   VideoSendStream* send_stream =
    413       call->CreateVideoSendStream(send_config, video_streams, NULL);
    414   analyzer.input_ = send_stream->Input();
    415 
    416   scoped_ptr<test::FrameGeneratorCapturer> file_capturer(
    417       test::FrameGeneratorCapturer::CreateFromYuvFile(
    418           &analyzer,
    419           test::ResourcePath(params.clip.name, "yuv").c_str(),
    420           params.clip.width,
    421           params.clip.height,
    422           params.clip.fps,
    423           Clock::GetRealTimeClock()));
    424   ASSERT_TRUE(file_capturer.get() != NULL)
    425       << "Could not create capturer for " << params.clip.name
    426       << ".yuv. Is this resource file present?";
    427 
    428   VideoReceiveStream::Config receive_config = call->GetDefaultReceiveConfig();
    429   VideoCodec codec =
    430       test::CreateDecoderVideoCodec(send_config.encoder_settings);
    431   receive_config.codecs.push_back(codec);
    432   receive_config.rtp.remote_ssrc = send_config.rtp.ssrcs[0];
    433   receive_config.rtp.local_ssrc = kReceiverLocalSsrc;
    434   receive_config.renderer = &analyzer;
    435 
    436   VideoReceiveStream* receive_stream =
    437       call->CreateVideoReceiveStream(receive_config);
    438 
    439   receive_stream->Start();
    440   send_stream->Start();
    441   file_capturer->Start();
    442 
    443   analyzer.Wait();
    444 
    445   file_capturer->Stop();
    446   send_stream->Stop();
    447   receive_stream->Stop();
    448 
    449   call->DestroyVideoReceiveStream(receive_stream);
    450   call->DestroyVideoSendStream(send_stream);
    451 
    452   transport.StopSending();
    453 }
    454 
    455 INSTANTIATE_TEST_CASE_P(FullStack,
    456                         FullStackTest,
    457                         ::testing::Values(paris_qcif, foreman_cif));
    458 
    459 }  // namespace webrtc
    460