1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include <sstream> 12 #include <string> 13 #include <vector> 14 15 #include "testing/gtest/include/gtest/gtest.h" 16 #include "webrtc/test/testsupport/fileutils.h" 17 #include "webrtc/test/testsupport/metrics/video_metrics.h" 18 #include "webrtc/test/testsupport/perf_test.h" 19 #include "webrtc/video_engine/test/auto_test/interface/vie_autotest.h" 20 #include "webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h" 21 #include "webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.h" 22 #include "webrtc/video_engine/test/libvietest/include/tb_external_transport.h" 23 #include "webrtc/video_engine/test/libvietest/include/vie_to_file_renderer.h" 24 25 namespace { 26 27 // The input file must be QCIF since I420 gets scaled to that in the tests 28 // (it is so bandwidth-heavy we have no choice). Our comparison algorithms 29 // wouldn't like scaling, so this will work when we compare with the original. 30 const int kInputWidth = 176; 31 const int kInputHeight = 144; 32 33 class ViEVideoVerificationTest : public testing::Test { 34 protected: 35 void SetUp() { 36 input_file_ = webrtc::test::ResourcePath("paris_qcif", "yuv"); 37 local_file_renderer_ = NULL; 38 remote_file_renderer_ = NULL; 39 } 40 41 void InitializeFileRenderers() { 42 local_file_renderer_ = new ViEToFileRenderer(); 43 remote_file_renderer_ = new ViEToFileRenderer(); 44 SetUpLocalFileRenderer(local_file_renderer_); 45 SetUpRemoteFileRenderer(remote_file_renderer_); 46 } 47 48 void SetUpLocalFileRenderer(ViEToFileRenderer* file_renderer) { 49 SetUpFileRenderer(file_renderer, "-local-preview.yuv"); 50 } 51 52 void SetUpRemoteFileRenderer(ViEToFileRenderer* file_renderer) { 53 SetUpFileRenderer(file_renderer, "-remote.yuv"); 54 } 55 56 // Must be called manually inside the tests. 57 void StopRenderers() { 58 local_file_renderer_->StopRendering(); 59 remote_file_renderer_->StopRendering(); 60 } 61 62 void CompareFiles(const std::string& reference_file, 63 const std::string& test_file, 64 double* psnr_result, double *ssim_result) { 65 webrtc::test::QualityMetricsResult psnr; 66 int error = I420PSNRFromFiles(reference_file.c_str(), test_file.c_str(), 67 kInputWidth, kInputHeight, &psnr); 68 69 EXPECT_EQ(0, error) << "PSNR routine failed - output files missing?"; 70 *psnr_result = psnr.average; 71 72 webrtc::test::QualityMetricsResult ssim; 73 error = I420SSIMFromFiles(reference_file.c_str(), test_file.c_str(), 74 kInputWidth, kInputHeight, &ssim); 75 EXPECT_EQ(0, error) << "SSIM routine failed - output files missing?"; 76 *ssim_result = ssim.average; 77 78 ViETest::Log("Results: PSNR is %f (dB; 48 is max), " 79 "SSIM is %f (1 is perfect)", 80 psnr.average, ssim.average); 81 } 82 83 // Note: must call AFTER CompareFiles. 84 void TearDownFileRenderers() { 85 TearDownFileRenderer(local_file_renderer_); 86 TearDownFileRenderer(remote_file_renderer_); 87 } 88 89 std::string input_file_; 90 ViEToFileRenderer* local_file_renderer_; 91 ViEToFileRenderer* remote_file_renderer_; 92 ViEFileBasedComparisonTests tests_; 93 94 private: 95 void SetUpFileRenderer(ViEToFileRenderer* file_renderer, 96 const std::string& suffix) { 97 std::string output_path = ViETest::GetResultOutputPath(); 98 std::string filename = "render_output" + suffix; 99 100 if (!file_renderer->PrepareForRendering(output_path, filename)) { 101 FAIL() << "Could not open output file " << filename << 102 " for writing."; 103 } 104 } 105 106 void TearDownFileRenderer(ViEToFileRenderer* file_renderer) { 107 assert(file_renderer); 108 bool test_failed = ::testing::UnitTest::GetInstance()-> 109 current_test_info()->result()->Failed(); 110 if (test_failed) { 111 // Leave the files for analysis if the test failed. 112 file_renderer->SaveOutputFile("failed-"); 113 } 114 delete file_renderer; 115 } 116 }; 117 118 class ParameterizedFullStackTest : public ViEVideoVerificationTest, 119 public ::testing::WithParamInterface<int> { 120 public: 121 static const int kNumFullStackInstances = 4; 122 123 protected: 124 struct TestParameters { 125 NetworkParameters network; 126 std::string file_name; 127 int width; 128 int height; 129 int bitrate; 130 double avg_psnr_threshold; 131 double avg_ssim_threshold; 132 ProtectionMethod protection_method; 133 std::string test_label; 134 }; 135 136 void SetUp() { 137 for (int i = 0; i < kNumFullStackInstances; ++i) { 138 parameter_table_[i].file_name = webrtc::test::ResourcePath("foreman_cif", 139 "yuv"); 140 parameter_table_[i].width = 352; 141 parameter_table_[i].height = 288; 142 } 143 int i = 0; 144 parameter_table_[i].protection_method = kNack; 145 // Uniform loss => Setting burst length to -1. 146 parameter_table_[i].network.loss_model = kUniformLoss; 147 parameter_table_[i].network.packet_loss_rate = 0; 148 parameter_table_[i].network.burst_length = -1; 149 parameter_table_[i].network.mean_one_way_delay = 0; 150 parameter_table_[i].network.std_dev_one_way_delay = 0; 151 parameter_table_[i].bitrate = 300; 152 // TODO(holmer): Enable for Win and Mac when the file rendering has been 153 // moved to a separate thread. 154 #ifdef WEBRTC_LINUX 155 parameter_table_[i].avg_psnr_threshold = 35; 156 parameter_table_[i].avg_ssim_threshold = 0.96; 157 #else 158 parameter_table_[i].avg_psnr_threshold = 0; 159 parameter_table_[i].avg_ssim_threshold = 0.0; 160 #endif 161 parameter_table_[i].test_label = "net_delay_0_0_plr_0"; 162 ++i; 163 parameter_table_[i].protection_method = kNack; 164 parameter_table_[i].network.loss_model = kUniformLoss; 165 parameter_table_[i].network.packet_loss_rate = 5; 166 parameter_table_[i].network.burst_length = -1; 167 parameter_table_[i].network.mean_one_way_delay = 50; 168 parameter_table_[i].network.std_dev_one_way_delay = 5; 169 parameter_table_[i].bitrate = 300; 170 // TODO(holmer): Enable for Win and Mac when the file rendering has been 171 // moved to a separate thread. 172 #ifdef WEBRTC_LINUX 173 parameter_table_[i].avg_psnr_threshold = 35; 174 parameter_table_[i].avg_ssim_threshold = 0.96; 175 #else 176 parameter_table_[i].avg_psnr_threshold = 0; 177 parameter_table_[i].avg_ssim_threshold = 0.0; 178 #endif 179 parameter_table_[i].test_label = "net_delay_50_5_plr_5"; 180 ++i; 181 parameter_table_[i].protection_method = kNack; 182 parameter_table_[i].network.loss_model = kUniformLoss; 183 parameter_table_[i].network.packet_loss_rate = 0; 184 parameter_table_[i].network.burst_length = -1; 185 parameter_table_[i].network.mean_one_way_delay = 100; 186 parameter_table_[i].network.std_dev_one_way_delay = 10; 187 parameter_table_[i].bitrate = 300; 188 // TODO(holmer): Enable for Win and Mac when the file rendering has been 189 // moved to a separate thread. 190 #ifdef WEBRTC_LINUX 191 parameter_table_[i].avg_psnr_threshold = 35; 192 parameter_table_[i].avg_ssim_threshold = 0.96; 193 #else 194 parameter_table_[i].avg_psnr_threshold = 0; 195 parameter_table_[i].avg_ssim_threshold = 0.0; 196 #endif 197 parameter_table_[i].test_label = "net_delay_100_10_plr_0"; 198 ++i; 199 parameter_table_[i].protection_method = kNack; 200 parameter_table_[i].network.loss_model = kGilbertElliotLoss; 201 parameter_table_[i].network.packet_loss_rate = 5; 202 parameter_table_[i].network.burst_length = 3; 203 parameter_table_[i].network.mean_one_way_delay = 100; 204 parameter_table_[i].network.std_dev_one_way_delay = 10; 205 parameter_table_[i].bitrate = 300; 206 // Thresholds disabled for now. This is being run mainly to get a graph. 207 parameter_table_[i].avg_psnr_threshold = 0; 208 parameter_table_[i].avg_ssim_threshold = 0.0; 209 parameter_table_[i].test_label = "net_delay_100_10_plr_5_gilbert_elliot"; 210 211 ASSERT_EQ(kNumFullStackInstances - 1, i); 212 } 213 214 TestParameters parameter_table_[kNumFullStackInstances]; 215 }; 216 217 TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) { 218 // I420 is lossless, so the I420 test should obviously get perfect results - 219 // the local preview and remote output files should be bit-exact. This test 220 // runs on external transport to ensure we do not drop packets. 221 // However, it's hard to make 100% stringent requirements on the video engine 222 // since for instance the jitter buffer has non-deterministic elements. If it 223 // breaks five times in a row though, you probably introduced a bug. 224 const double kReasonablePsnr = webrtc::test::kMetricsPerfectPSNR - 2.0f; 225 const double kReasonableSsim = 0.99f; 226 const int kNumAttempts = 5; 227 for (int attempt = 0; attempt < kNumAttempts; ++attempt) { 228 InitializeFileRenderers(); 229 ASSERT_TRUE(tests_.TestCallSetup(input_file_, kInputWidth, kInputHeight, 230 local_file_renderer_, 231 remote_file_renderer_)); 232 std::string remote_file = remote_file_renderer_->GetFullOutputPath(); 233 std::string local_preview = local_file_renderer_->GetFullOutputPath(); 234 StopRenderers(); 235 236 double actual_psnr = 0; 237 double actual_ssim = 0; 238 CompareFiles(local_preview, remote_file, &actual_psnr, &actual_ssim); 239 240 TearDownFileRenderers(); 241 242 if (actual_psnr > kReasonablePsnr && actual_ssim > kReasonableSsim) { 243 // Test successful. 244 return; 245 } else { 246 ViETest::Log("Retrying; attempt %d of %d.", attempt + 1, kNumAttempts); 247 } 248 } 249 250 FAIL() << "Failed to achieve near-perfect PSNR and SSIM results after " << 251 kNumAttempts << " attempts."; 252 } 253 254 // Runs a whole stack processing with tracking of which frames are dropped 255 // in the encoder. Tests show that they start at the same frame, which is 256 // the important thing when doing frame-to-frame comparison with PSNR/SSIM. 257 TEST_P(ParameterizedFullStackTest, RunsFullStackWithoutErrors) { 258 // Using CIF here since it's a more common resolution than QCIF, and higher 259 // resolutions shouldn't be a problem for a test using VP8. 260 input_file_ = parameter_table_[GetParam()].file_name; 261 FrameDropDetector detector; 262 local_file_renderer_ = new ViEToFileRenderer(); 263 remote_file_renderer_ = new FrameDropMonitoringRemoteFileRenderer(&detector); 264 SetUpLocalFileRenderer(local_file_renderer_); 265 SetUpRemoteFileRenderer(remote_file_renderer_); 266 267 // Set a low bit rate so the encoder budget will be tight, causing it to drop 268 // frames every now and then. 269 const int kBitRateKbps = parameter_table_[GetParam()].bitrate; 270 const NetworkParameters network = parameter_table_[GetParam()].network; 271 int width = parameter_table_[GetParam()].width; 272 int height = parameter_table_[GetParam()].height; 273 ProtectionMethod protection_method = 274 parameter_table_[GetParam()].protection_method; 275 ViETest::Log("Bit rate : %5d kbps", kBitRateKbps); 276 ViETest::Log("Packet loss : %5d %%", network.packet_loss_rate); 277 ViETest::Log("Network delay: mean=%dms std dev=%d ms", 278 network.mean_one_way_delay, network.std_dev_one_way_delay); 279 tests_.TestFullStack(input_file_, width, height, kBitRateKbps, 280 protection_method, network, local_file_renderer_, 281 remote_file_renderer_, &detector); 282 const std::string reference_file = local_file_renderer_->GetFullOutputPath(); 283 const std::string output_file = remote_file_renderer_->GetFullOutputPath(); 284 StopRenderers(); 285 286 detector.CalculateResults(); 287 detector.PrintReport(parameter_table_[GetParam()].test_label); 288 289 if (detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered) > 290 detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded)) { 291 detector.PrintDebugDump(); 292 } 293 294 ASSERT_GE(detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered), 295 detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded)) 296 << "The number of dropped frames on the decode and render steps are not " 297 "equal. This may be because we have a major problem in the buffers of " 298 "the ViEToFileRenderer?"; 299 300 // We may have dropped frames during the processing, which means the output 301 // file does not contain all the frames that are present in the input file. 302 // To make the quality measurement correct, we must adjust the output file to 303 // that by copying the last successful frame into the place where the dropped 304 // frame would be, for all dropped frames. 305 const int frame_length_in_bytes = 3 * width * height / 2; 306 ViETest::Log("Frame length: %d bytes", frame_length_in_bytes); 307 std::vector<Frame*> all_frames = detector.GetAllFrames(); 308 FixOutputFileForComparison(output_file, frame_length_in_bytes, all_frames); 309 310 // Verify all sent frames are present in the output file. 311 size_t output_file_size = webrtc::test::GetFileSize(output_file); 312 EXPECT_EQ(all_frames.size(), output_file_size / frame_length_in_bytes) 313 << "The output file size is incorrect. It should be equal to the number " 314 "of frames multiplied by the frame size. This will likely affect " 315 "PSNR/SSIM calculations in a bad way."; 316 317 TearDownFileRenderers(); 318 319 // We are running on a lower bitrate here so we need to settle for somewhat 320 // lower PSNR and SSIM values. 321 double actual_psnr = 0; 322 double actual_ssim = 0; 323 CompareFiles(reference_file, output_file, &actual_psnr, &actual_ssim); 324 325 const double kExpectedMinimumPSNR = 326 parameter_table_[GetParam()].avg_psnr_threshold; 327 const double kExpectedMinimumSSIM = 328 parameter_table_[GetParam()].avg_ssim_threshold; 329 330 EXPECT_GE(actual_psnr, kExpectedMinimumPSNR); 331 EXPECT_GE(actual_ssim, kExpectedMinimumSSIM); 332 333 std::stringstream ss; 334 ss << std::setprecision(3) << std::fixed << actual_psnr; 335 webrtc::test::PrintResult( 336 "psnr", "", parameter_table_[GetParam()].test_label, 337 ss.str(), "dB", false); 338 339 ss.str(""); 340 ss << std::setprecision(3) << std::fixed << actual_ssim; 341 webrtc::test::PrintResult( 342 "ssim", "", parameter_table_[GetParam()].test_label, 343 ss.str(), "", false); 344 } 345 346 INSTANTIATE_TEST_CASE_P(FullStackTests, ParameterizedFullStackTest, 347 ::testing::Range(0, ParameterizedFullStackTest::kNumFullStackInstances)); 348 349 } // namespace 350
