1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include <math.h> 12 13 #include "testing/gtest/include/gtest/gtest.h" 14 15 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" 16 #include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h" 17 #include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" 18 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" 19 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" 20 #include "webrtc/modules/video_coding/main/interface/video_coding.h" 21 #include "webrtc/test/testsupport/fileutils.h" 22 #include "webrtc/test/testsupport/frame_reader.h" 23 #include "webrtc/test/testsupport/frame_writer.h" 24 #include "webrtc/test/testsupport/gtest_disable.h" 25 #include "webrtc/test/testsupport/metrics/video_metrics.h" 26 #include "webrtc/test/testsupport/packet_reader.h" 27 #include "webrtc/typedefs.h" 28 29 namespace webrtc { 30 31 // Maximum number of rate updates (i.e., calls to encoder to change bitrate 32 // and/or frame rate) for the current tests. 33 const int kMaxNumRateUpdates = 3; 34 35 const int kPercTargetvsActualMismatch = 20; 36 const int kBaseKeyFrameInterval = 3000; 37 38 // Codec and network settings. 39 struct CodecConfigPars { 40 float packet_loss; 41 int num_temporal_layers; 42 int key_frame_interval; 43 bool error_concealment_on; 44 bool denoising_on; 45 bool frame_dropper_on; 46 bool spatial_resize_on; 47 }; 48 49 // Quality metrics. 50 struct QualityMetrics { 51 double minimum_avg_psnr; 52 double minimum_min_psnr; 53 double minimum_avg_ssim; 54 double minimum_min_ssim; 55 }; 56 57 // The sequence of bitrate and frame rate changes for the encoder, the frame 58 // number where the changes are made, and the total number of frames for the 59 // test. 60 struct RateProfile { 61 int target_bit_rate[kMaxNumRateUpdates]; 62 int input_frame_rate[kMaxNumRateUpdates]; 63 int frame_index_rate_update[kMaxNumRateUpdates + 1]; 64 int num_frames; 65 }; 66 67 // Metrics for the rate control. The rate mismatch metrics are defined as 68 // percentages.|max_time_hit_target| is defined as number of frames, after a 69 // rate update is made to the encoder, for the encoder to reach within 70 // |kPercTargetvsActualMismatch| of new target rate. The metrics are defined for 71 // each rate update sequence. 72 struct RateControlMetrics { 73 int max_num_dropped_frames; 74 int max_key_frame_size_mismatch; 75 int max_delta_frame_size_mismatch; 76 int max_encoding_rate_mismatch; 77 int max_time_hit_target; 78 int num_spatial_resizes; 79 }; 80 81 82 // Sequence used is foreman (CIF): may be better to use VGA for resize test. 83 const int kCIFWidth = 352; 84 const int kCIFHeight = 288; 85 const int kNbrFramesShort = 100; // Some tests are run for shorter sequence. 86 const int kNbrFramesLong = 299; 87 88 // Parameters from VP8 wrapper, which control target size of key frames. 89 const float kInitialBufferSize = 0.5f; 90 const float kOptimalBufferSize = 0.6f; 91 const float kScaleKeyFrameSize = 0.5f; 92 93 // Integration test for video processor. Encodes+decodes a clip and 94 // writes it to the output directory. After completion, quality metrics 95 // (PSNR and SSIM) and rate control metrics are computed to verify that the 96 // quality and encoder response is acceptable. The rate control tests allow us 97 // to verify the behavior for changing bitrate, changing frame rate, frame 98 // dropping/spatial resize, and temporal layers. The limits for the rate 99 // control metrics are set to be fairly conservative, so failure should only 100 // happen when some significant regression or breakdown occurs. 101 class VideoProcessorIntegrationTest: public testing::Test { 102 protected: 103 VideoEncoder* encoder_; 104 VideoDecoder* decoder_; 105 webrtc::test::FrameReader* frame_reader_; 106 webrtc::test::FrameWriter* frame_writer_; 107 webrtc::test::PacketReader packet_reader_; 108 webrtc::test::PacketManipulator* packet_manipulator_; 109 webrtc::test::Stats stats_; 110 webrtc::test::TestConfig config_; 111 VideoCodec codec_settings_; 112 webrtc::test::VideoProcessor* processor_; 113 114 // Quantities defined/updated for every encoder rate update. 115 // Some quantities defined per temporal layer (at most 3 layers in this test). 116 int num_frames_per_update_[3]; 117 float sum_frame_size_mismatch_[3]; 118 float sum_encoded_frame_size_[3]; 119 float encoding_bitrate_[3]; 120 float per_frame_bandwidth_[3]; 121 float bit_rate_layer_[3]; 122 float frame_rate_layer_[3]; 123 int num_frames_total_; 124 float sum_encoded_frame_size_total_; 125 float encoding_bitrate_total_; 126 float perc_encoding_rate_mismatch_; 127 int num_frames_to_hit_target_; 128 bool encoding_rate_within_target_; 129 int bit_rate_; 130 int frame_rate_; 131 int layer_; 132 float target_size_key_frame_initial_; 133 float target_size_key_frame_; 134 float sum_key_frame_size_mismatch_; 135 int num_key_frames_; 136 float start_bitrate_; 137 138 // Codec and network settings. 139 float packet_loss_; 140 int num_temporal_layers_; 141 int key_frame_interval_; 142 bool error_concealment_on_; 143 bool denoising_on_; 144 bool frame_dropper_on_; 145 bool spatial_resize_on_; 146 147 148 VideoProcessorIntegrationTest() {} 149 virtual ~VideoProcessorIntegrationTest() {} 150 151 void SetUpCodecConfig() { 152 encoder_ = VP8Encoder::Create(); 153 decoder_ = VP8Decoder::Create(); 154 155 // CIF is currently used for all tests below. 156 // Setup the TestConfig struct for processing of a clip in CIF resolution. 157 config_.input_filename = 158 webrtc::test::ResourcePath("foreman_cif", "yuv"); 159 160 // Generate an output filename in a safe way. 161 config_.output_filename = webrtc::test::TempFilename( 162 webrtc::test::OutputPath(), "videoprocessor_integrationtest"); 163 config_.frame_length_in_bytes = CalcBufferSize(kI420, 164 kCIFWidth, kCIFHeight); 165 config_.verbose = false; 166 // Only allow encoder/decoder to use single core, for predictability. 167 config_.use_single_core = true; 168 // Key frame interval and packet loss are set for each test. 169 config_.keyframe_interval = key_frame_interval_; 170 config_.networking_config.packet_loss_probability = packet_loss_; 171 172 // Get a codec configuration struct and configure it. 173 VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_); 174 config_.codec_settings = &codec_settings_; 175 config_.codec_settings->startBitrate = start_bitrate_; 176 config_.codec_settings->width = kCIFWidth; 177 config_.codec_settings->height = kCIFHeight; 178 // These features may be set depending on the test. 179 config_.codec_settings->codecSpecific.VP8.errorConcealmentOn = 180 error_concealment_on_; 181 config_.codec_settings->codecSpecific.VP8.denoisingOn = 182 denoising_on_; 183 config_.codec_settings->codecSpecific.VP8.numberOfTemporalLayers = 184 num_temporal_layers_; 185 config_.codec_settings->codecSpecific.VP8.frameDroppingOn = 186 frame_dropper_on_; 187 config_.codec_settings->codecSpecific.VP8.automaticResizeOn = 188 spatial_resize_on_; 189 config_.codec_settings->codecSpecific.VP8.keyFrameInterval = 190 kBaseKeyFrameInterval; 191 192 frame_reader_ = 193 new webrtc::test::FrameReaderImpl(config_.input_filename, 194 config_.frame_length_in_bytes); 195 frame_writer_ = 196 new webrtc::test::FrameWriterImpl(config_.output_filename, 197 config_.frame_length_in_bytes); 198 ASSERT_TRUE(frame_reader_->Init()); 199 ASSERT_TRUE(frame_writer_->Init()); 200 201 packet_manipulator_ = new webrtc::test::PacketManipulatorImpl( 202 &packet_reader_, config_.networking_config, config_.verbose); 203 processor_ = new webrtc::test::VideoProcessorImpl(encoder_, decoder_, 204 frame_reader_, 205 frame_writer_, 206 packet_manipulator_, 207 config_, &stats_); 208 ASSERT_TRUE(processor_->Init()); 209 } 210 211 // Reset quantities after each encoder update, update the target 212 // per-frame bandwidth. 213 void ResetRateControlMetrics(int num_frames) { 214 for (int i = 0; i < num_temporal_layers_; i++) { 215 num_frames_per_update_[i] = 0; 216 sum_frame_size_mismatch_[i] = 0.0f; 217 sum_encoded_frame_size_[i] = 0.0f; 218 encoding_bitrate_[i] = 0.0f; 219 // Update layer per-frame-bandwidth. 220 per_frame_bandwidth_[i] = static_cast<float>(bit_rate_layer_[i]) / 221 static_cast<float>(frame_rate_layer_[i]); 222 } 223 // Set maximum size of key frames, following setting in the VP8 wrapper. 224 float max_key_size = kScaleKeyFrameSize * kOptimalBufferSize * frame_rate_; 225 // We don't know exact target size of the key frames (except for first one), 226 // but the minimum in libvpx is ~|3 * per_frame_bandwidth| and maximum is 227 // set by |max_key_size_ * per_frame_bandwidth|. Take middle point/average 228 // as reference for mismatch. Note key frames always correspond to base 229 // layer frame in this test. 230 target_size_key_frame_ = 0.5 * (3 + max_key_size) * per_frame_bandwidth_[0]; 231 num_frames_total_ = 0; 232 sum_encoded_frame_size_total_ = 0.0f; 233 encoding_bitrate_total_ = 0.0f; 234 perc_encoding_rate_mismatch_ = 0.0f; 235 num_frames_to_hit_target_ = num_frames; 236 encoding_rate_within_target_ = false; 237 sum_key_frame_size_mismatch_ = 0.0; 238 num_key_frames_ = 0; 239 } 240 241 // For every encoded frame, update the rate control metrics. 242 void UpdateRateControlMetrics(int frame_num, VideoFrameType frame_type) { 243 int encoded_frame_size = processor_->EncodedFrameSize(); 244 float encoded_size_kbits = encoded_frame_size * 8.0f / 1000.0f; 245 // Update layer data. 246 // Update rate mismatch relative to per-frame bandwidth for delta frames. 247 if (frame_type == kDeltaFrame) { 248 // TODO(marpan): Should we count dropped (zero size) frames in mismatch? 249 sum_frame_size_mismatch_[layer_] += fabs(encoded_size_kbits - 250 per_frame_bandwidth_[layer_]) / 251 per_frame_bandwidth_[layer_]; 252 } else { 253 float target_size = (frame_num == 1) ? target_size_key_frame_initial_ : 254 target_size_key_frame_; 255 sum_key_frame_size_mismatch_ += fabs(encoded_size_kbits - target_size) / 256 target_size; 257 num_key_frames_ += 1; 258 } 259 sum_encoded_frame_size_[layer_] += encoded_size_kbits; 260 // Encoding bitrate per layer: from the start of the update/run to the 261 // current frame. 262 encoding_bitrate_[layer_] = sum_encoded_frame_size_[layer_] * 263 frame_rate_layer_[layer_] / 264 num_frames_per_update_[layer_]; 265 // Total encoding rate: from the start of the update/run to current frame. 266 sum_encoded_frame_size_total_ += encoded_size_kbits; 267 encoding_bitrate_total_ = sum_encoded_frame_size_total_ * frame_rate_ / 268 num_frames_total_; 269 perc_encoding_rate_mismatch_ = 100 * fabs(encoding_bitrate_total_ - 270 bit_rate_) / bit_rate_; 271 if (perc_encoding_rate_mismatch_ < kPercTargetvsActualMismatch && 272 !encoding_rate_within_target_) { 273 num_frames_to_hit_target_ = num_frames_total_; 274 encoding_rate_within_target_ = true; 275 } 276 } 277 278 // Verify expected behavior of rate control and print out data. 279 void VerifyRateControl(int update_index, 280 int max_key_frame_size_mismatch, 281 int max_delta_frame_size_mismatch, 282 int max_encoding_rate_mismatch, 283 int max_time_hit_target, 284 int max_num_dropped_frames, 285 int num_spatial_resizes) { 286 int num_dropped_frames = processor_->NumberDroppedFrames(); 287 int num_resize_actions = processor_->NumberSpatialResizes(); 288 printf("For update #: %d,\n " 289 " Target Bitrate: %d,\n" 290 " Encoding bitrate: %f,\n" 291 " Frame rate: %d \n", 292 update_index, bit_rate_, encoding_bitrate_total_, frame_rate_); 293 printf(" Number of frames to approach target rate = %d, \n" 294 " Number of dropped frames = %d, \n" 295 " Number of spatial resizes = %d, \n", 296 num_frames_to_hit_target_, num_dropped_frames, num_resize_actions); 297 EXPECT_LE(perc_encoding_rate_mismatch_, max_encoding_rate_mismatch); 298 if (num_key_frames_ > 0) { 299 int perc_key_frame_size_mismatch = 100 * sum_key_frame_size_mismatch_ / 300 num_key_frames_; 301 printf(" Number of Key frames: %d \n" 302 " Key frame rate mismatch: %d \n", 303 num_key_frames_, perc_key_frame_size_mismatch); 304 EXPECT_LE(perc_key_frame_size_mismatch, max_key_frame_size_mismatch); 305 } 306 printf("\n"); 307 printf("Rates statistics for Layer data \n"); 308 for (int i = 0; i < num_temporal_layers_ ; i++) { 309 printf("Layer #%d \n", i); 310 int perc_frame_size_mismatch = 100 * sum_frame_size_mismatch_[i] / 311 num_frames_per_update_[i]; 312 int perc_encoding_rate_mismatch = 100 * fabs(encoding_bitrate_[i] - 313 bit_rate_layer_[i]) / 314 bit_rate_layer_[i]; 315 printf(" Target Layer Bit rate: %f \n" 316 " Layer frame rate: %f, \n" 317 " Layer per frame bandwidth: %f, \n" 318 " Layer Encoding bit rate: %f, \n" 319 " Layer Percent frame size mismatch: %d, \n" 320 " Layer Percent encoding rate mismatch = %d, \n" 321 " Number of frame processed per layer = %d \n", 322 bit_rate_layer_[i], frame_rate_layer_[i], per_frame_bandwidth_[i], 323 encoding_bitrate_[i], perc_frame_size_mismatch, 324 perc_encoding_rate_mismatch, num_frames_per_update_[i]); 325 EXPECT_LE(perc_frame_size_mismatch, max_delta_frame_size_mismatch); 326 EXPECT_LE(perc_encoding_rate_mismatch, max_encoding_rate_mismatch); 327 } 328 printf("\n"); 329 EXPECT_LE(num_frames_to_hit_target_, max_time_hit_target); 330 EXPECT_LE(num_dropped_frames, max_num_dropped_frames); 331 EXPECT_EQ(num_resize_actions, num_spatial_resizes); 332 } 333 334 // Layer index corresponding to frame number, for up to 3 layers. 335 void LayerIndexForFrame(int frame_number) { 336 if (num_temporal_layers_ == 1) { 337 layer_ = 0; 338 } else if (num_temporal_layers_ == 2) { 339 // layer 0: 0 2 4 ... 340 // layer 1: 1 3 341 if (frame_number % 2 == 0) { 342 layer_ = 0; 343 } else { 344 layer_ = 1; 345 } 346 } else if (num_temporal_layers_ == 3) { 347 // layer 0: 0 4 8 ... 348 // layer 1: 2 6 349 // layer 2: 1 3 5 7 350 if (frame_number % 4 == 0) { 351 layer_ = 0; 352 } else if ((frame_number + 2) % 4 == 0) { 353 layer_ = 1; 354 } else if ((frame_number + 1) % 2 == 0) { 355 layer_ = 2; 356 } 357 } else { 358 assert(false); // Only up to 3 layers. 359 } 360 } 361 362 // Set the bitrate and frame rate per layer, for up to 3 layers. 363 void SetLayerRates() { 364 assert(num_temporal_layers_<= 3); 365 for (int i = 0; i < num_temporal_layers_; i++) { 366 float bit_rate_ratio = 367 kVp8LayerRateAlloction[num_temporal_layers_ - 1][i]; 368 if (i > 0) { 369 float bit_rate_delta_ratio = kVp8LayerRateAlloction 370 [num_temporal_layers_ - 1][i] - 371 kVp8LayerRateAlloction[num_temporal_layers_ - 1][i - 1]; 372 bit_rate_layer_[i] = bit_rate_ * bit_rate_delta_ratio; 373 } else { 374 bit_rate_layer_[i] = bit_rate_ * bit_rate_ratio; 375 } 376 frame_rate_layer_[i] = frame_rate_ / static_cast<float>( 377 1 << (num_temporal_layers_ - 1)); 378 } 379 if (num_temporal_layers_ == 3) { 380 frame_rate_layer_[2] = frame_rate_ / 2.0f; 381 } 382 } 383 384 VideoFrameType FrameType(int frame_number) { 385 if (frame_number == 0 || ((frame_number) % key_frame_interval_ == 0 && 386 key_frame_interval_ > 0)) { 387 return kKeyFrame; 388 } else { 389 return kDeltaFrame; 390 } 391 } 392 393 void TearDown() { 394 delete processor_; 395 delete packet_manipulator_; 396 delete frame_writer_; 397 delete frame_reader_; 398 delete decoder_; 399 delete encoder_; 400 } 401 402 // Processes all frames in the clip and verifies the result. 403 void ProcessFramesAndVerify(QualityMetrics quality_metrics, 404 RateProfile rate_profile, 405 CodecConfigPars process, 406 RateControlMetrics* rc_metrics) { 407 // Codec/config settings. 408 start_bitrate_ = rate_profile.target_bit_rate[0]; 409 packet_loss_ = process.packet_loss; 410 key_frame_interval_ = process.key_frame_interval; 411 num_temporal_layers_ = process.num_temporal_layers; 412 error_concealment_on_ = process.error_concealment_on; 413 denoising_on_ = process.denoising_on; 414 frame_dropper_on_ = process.frame_dropper_on; 415 spatial_resize_on_ = process.spatial_resize_on; 416 SetUpCodecConfig(); 417 // Update the layers and the codec with the initial rates. 418 bit_rate_ = rate_profile.target_bit_rate[0]; 419 frame_rate_ = rate_profile.input_frame_rate[0]; 420 SetLayerRates(); 421 // Set the initial target size for key frame. 422 target_size_key_frame_initial_ = 0.5 * kInitialBufferSize * 423 bit_rate_layer_[0]; 424 processor_->SetRates(bit_rate_, frame_rate_); 425 // Process each frame, up to |num_frames|. 426 int num_frames = rate_profile.num_frames; 427 int update_index = 0; 428 ResetRateControlMetrics( 429 rate_profile.frame_index_rate_update[update_index + 1]); 430 int frame_number = 0; 431 VideoFrameType frame_type = kDeltaFrame; 432 while (processor_->ProcessFrame(frame_number) && 433 frame_number < num_frames) { 434 // Get the layer index for the frame |frame_number|. 435 LayerIndexForFrame(frame_number); 436 frame_type = FrameType(frame_number); 437 // Counter for whole sequence run. 438 ++frame_number; 439 // Counters for each rate update. 440 ++num_frames_per_update_[layer_]; 441 ++num_frames_total_; 442 UpdateRateControlMetrics(frame_number, frame_type); 443 // If we hit another/next update, verify stats for current state and 444 // update layers and codec with new rates. 445 if (frame_number == 446 rate_profile.frame_index_rate_update[update_index + 1]) { 447 VerifyRateControl( 448 update_index, 449 rc_metrics[update_index].max_key_frame_size_mismatch, 450 rc_metrics[update_index].max_delta_frame_size_mismatch, 451 rc_metrics[update_index].max_encoding_rate_mismatch, 452 rc_metrics[update_index].max_time_hit_target, 453 rc_metrics[update_index].max_num_dropped_frames, 454 rc_metrics[update_index].num_spatial_resizes); 455 // Update layer rates and the codec with new rates. 456 ++update_index; 457 bit_rate_ = rate_profile.target_bit_rate[update_index]; 458 frame_rate_ = rate_profile.input_frame_rate[update_index]; 459 SetLayerRates(); 460 ResetRateControlMetrics(rate_profile. 461 frame_index_rate_update[update_index + 1]); 462 processor_->SetRates(bit_rate_, frame_rate_); 463 } 464 } 465 VerifyRateControl( 466 update_index, 467 rc_metrics[update_index].max_key_frame_size_mismatch, 468 rc_metrics[update_index].max_delta_frame_size_mismatch, 469 rc_metrics[update_index].max_encoding_rate_mismatch, 470 rc_metrics[update_index].max_time_hit_target, 471 rc_metrics[update_index].max_num_dropped_frames, 472 rc_metrics[update_index].num_spatial_resizes); 473 EXPECT_EQ(num_frames, frame_number); 474 EXPECT_EQ(num_frames + 1, static_cast<int>(stats_.stats_.size())); 475 476 // Release encoder and decoder to make sure they have finished processing: 477 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); 478 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release()); 479 // Close the files before we start using them for SSIM/PSNR calculations. 480 frame_reader_->Close(); 481 frame_writer_->Close(); 482 483 // TODO(marpan): should compute these quality metrics per SetRates update. 484 webrtc::test::QualityMetricsResult psnr_result, ssim_result; 485 EXPECT_EQ(0, webrtc::test::I420MetricsFromFiles( 486 config_.input_filename.c_str(), 487 config_.output_filename.c_str(), 488 config_.codec_settings->width, 489 config_.codec_settings->height, 490 &psnr_result, 491 &ssim_result)); 492 printf("PSNR avg: %f, min: %f SSIM avg: %f, min: %f\n", 493 psnr_result.average, psnr_result.min, 494 ssim_result.average, ssim_result.min); 495 stats_.PrintSummary(); 496 EXPECT_GT(psnr_result.average, quality_metrics.minimum_avg_psnr); 497 EXPECT_GT(psnr_result.min, quality_metrics.minimum_min_psnr); 498 EXPECT_GT(ssim_result.average, quality_metrics.minimum_avg_ssim); 499 EXPECT_GT(ssim_result.min, quality_metrics.minimum_min_ssim); 500 if (!remove(config_.output_filename.c_str())) { 501 fprintf(stderr, "Failed to remove temporary file!"); 502 } 503 } 504 }; 505 506 void SetRateProfilePars(RateProfile* rate_profile, 507 int update_index, 508 int bit_rate, 509 int frame_rate, 510 int frame_index_rate_update) { 511 rate_profile->target_bit_rate[update_index] = bit_rate; 512 rate_profile->input_frame_rate[update_index] = frame_rate; 513 rate_profile->frame_index_rate_update[update_index] = frame_index_rate_update; 514 } 515 516 void SetCodecParameters(CodecConfigPars* process_settings, 517 float packet_loss, 518 int key_frame_interval, 519 int num_temporal_layers, 520 bool error_concealment_on, 521 bool denoising_on, 522 bool frame_dropper_on, 523 bool spatial_resize_on) { 524 process_settings->packet_loss = packet_loss; 525 process_settings->key_frame_interval = key_frame_interval; 526 process_settings->num_temporal_layers = num_temporal_layers, 527 process_settings->error_concealment_on = error_concealment_on; 528 process_settings->denoising_on = denoising_on; 529 process_settings->frame_dropper_on = frame_dropper_on; 530 process_settings->spatial_resize_on = spatial_resize_on; 531 } 532 533 void SetQualityMetrics(QualityMetrics* quality_metrics, 534 double minimum_avg_psnr, 535 double minimum_min_psnr, 536 double minimum_avg_ssim, 537 double minimum_min_ssim) { 538 quality_metrics->minimum_avg_psnr = minimum_avg_psnr; 539 quality_metrics->minimum_min_psnr = minimum_min_psnr; 540 quality_metrics->minimum_avg_ssim = minimum_avg_ssim; 541 quality_metrics->minimum_min_ssim = minimum_min_ssim; 542 } 543 544 void SetRateControlMetrics(RateControlMetrics* rc_metrics, 545 int update_index, 546 int max_num_dropped_frames, 547 int max_key_frame_size_mismatch, 548 int max_delta_frame_size_mismatch, 549 int max_encoding_rate_mismatch, 550 int max_time_hit_target, 551 int num_spatial_resizes) { 552 rc_metrics[update_index].max_num_dropped_frames = max_num_dropped_frames; 553 rc_metrics[update_index].max_key_frame_size_mismatch = 554 max_key_frame_size_mismatch; 555 rc_metrics[update_index].max_delta_frame_size_mismatch = 556 max_delta_frame_size_mismatch; 557 rc_metrics[update_index].max_encoding_rate_mismatch = 558 max_encoding_rate_mismatch; 559 rc_metrics[update_index].max_time_hit_target = max_time_hit_target; 560 rc_metrics[update_index].num_spatial_resizes = num_spatial_resizes; 561 } 562 563 // Run with no packet loss and fixed bitrate. Quality should be very high. 564 // One key frame (first frame only) in sequence. Setting |key_frame_interval| 565 // to -1 below means no periodic key frames in test. 566 TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) { 567 // Bitrate and frame rate profile. 568 RateProfile rate_profile; 569 SetRateProfilePars(&rate_profile, 0, 500, 30, 0); 570 rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; 571 rate_profile.num_frames = kNbrFramesShort; 572 // Codec/network settings. 573 CodecConfigPars process_settings; 574 SetCodecParameters(&process_settings, 0.0f, -1, 1, false, true, true, false); 575 // Metrics for expected quality. 576 QualityMetrics quality_metrics; 577 SetQualityMetrics(&quality_metrics, 34.95, 33.0, 0.90, 0.89); 578 // Metrics for rate control. 579 RateControlMetrics rc_metrics[1]; 580 SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 15, 0); 581 ProcessFramesAndVerify(quality_metrics, 582 rate_profile, 583 process_settings, 584 rc_metrics); 585 } 586 587 // Run with 5% packet loss and fixed bitrate. Quality should be a bit lower. 588 // One key frame (first frame only) in sequence. 589 TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) { 590 // Bitrate and frame rate profile. 591 RateProfile rate_profile; 592 SetRateProfilePars(&rate_profile, 0, 500, 30, 0); 593 rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; 594 rate_profile.num_frames = kNbrFramesShort; 595 // Codec/network settings. 596 CodecConfigPars process_settings; 597 SetCodecParameters(&process_settings, 0.05f, -1, 1, false, true, true, false); 598 // Metrics for expected quality. 599 QualityMetrics quality_metrics; 600 SetQualityMetrics(&quality_metrics, 20.0, 16.0, 0.60, 0.40); 601 // Metrics for rate control. 602 RateControlMetrics rc_metrics[1]; 603 SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 15, 0); 604 ProcessFramesAndVerify(quality_metrics, 605 rate_profile, 606 process_settings, 607 rc_metrics); 608 } 609 610 // Run with 10% packet loss and fixed bitrate. Quality should be even lower. 611 // One key frame (first frame only) in sequence. 612 TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) { 613 // Bitrate and frame rate profile. 614 RateProfile rate_profile; 615 SetRateProfilePars(&rate_profile, 0, 500, 30, 0); 616 rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; 617 rate_profile.num_frames = kNbrFramesShort; 618 // Codec/network settings. 619 CodecConfigPars process_settings; 620 SetCodecParameters(&process_settings, 0.1f, -1, 1, false, true, true, false); 621 // Metrics for expected quality. 622 QualityMetrics quality_metrics; 623 SetQualityMetrics(&quality_metrics, 19.0, 16.0, 0.50, 0.35); 624 // Metrics for rate control. 625 RateControlMetrics rc_metrics[1]; 626 SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 15, 0); 627 ProcessFramesAndVerify(quality_metrics, 628 rate_profile, 629 process_settings, 630 rc_metrics); 631 } 632 633 // The tests below are currently disabled for Android. For ARM, the encoder 634 // uses |cpu_speed| = 12, as opposed to default |cpu_speed| <= 6 for x86, 635 // which leads to significantly different quality. The quality and rate control 636 // settings in the tests below are defined for encoder speed setting 637 // |cpu_speed| <= ~6. A number of settings would need to be significantly 638 // modified for the |cpu_speed| = 12 case. For now, keep the tests below 639 // disabled on Android. Some quality parameter in the above test has been 640 // adjusted to also pass for |cpu_speed| <= 12. 641 642 // Run with no packet loss, with varying bitrate (3 rate updates): 643 // low to high to medium. Check that quality and encoder response to the new 644 // target rate/per-frame bandwidth (for each rate update) is within limits. 645 // One key frame (first frame only) in sequence. 646 TEST_F(VideoProcessorIntegrationTest, 647 DISABLED_ON_ANDROID(ProcessNoLossChangeBitRate)) { 648 // Bitrate and frame rate profile. 649 RateProfile rate_profile; 650 SetRateProfilePars(&rate_profile, 0, 200, 30, 0); 651 SetRateProfilePars(&rate_profile, 1, 800, 30, 100); 652 SetRateProfilePars(&rate_profile, 2, 500, 30, 200); 653 rate_profile.frame_index_rate_update[3] = kNbrFramesLong + 1; 654 rate_profile.num_frames = kNbrFramesLong; 655 // Codec/network settings. 656 CodecConfigPars process_settings; 657 SetCodecParameters(&process_settings, 0.0f, -1, 1, false, true, true, false); 658 // Metrics for expected quality. 659 QualityMetrics quality_metrics; 660 SetQualityMetrics(&quality_metrics, 34.0, 32.0, 0.85, 0.80); 661 // Metrics for rate control. 662 RateControlMetrics rc_metrics[3]; 663 SetRateControlMetrics(rc_metrics, 0, 0, 45, 20, 10, 15, 0); 664 SetRateControlMetrics(rc_metrics, 1, 0, 0, 25, 20, 10, 0); 665 SetRateControlMetrics(rc_metrics, 2, 0, 0, 25, 15, 10, 0); 666 ProcessFramesAndVerify(quality_metrics, 667 rate_profile, 668 process_settings, 669 rc_metrics); 670 } 671 672 // Run with no packet loss, with an update (decrease) in frame rate. 673 // Lower frame rate means higher per-frame-bandwidth, so easier to encode. 674 // At the bitrate in this test, this means better rate control after the 675 // update(s) to lower frame rate. So expect less frame drops, and max values 676 // for the rate control metrics can be lower. One key frame (first frame only). 677 // Note: quality after update should be higher but we currently compute quality 678 // metrics avergaed over whole sequence run. 679 TEST_F(VideoProcessorIntegrationTest, 680 DISABLED_ON_ANDROID(ProcessNoLossChangeFrameRateFrameDrop)) { 681 config_.networking_config.packet_loss_probability = 0; 682 // Bitrate and frame rate profile. 683 RateProfile rate_profile; 684 SetRateProfilePars(&rate_profile, 0, 80, 24, 0); 685 SetRateProfilePars(&rate_profile, 1, 80, 15, 100); 686 SetRateProfilePars(&rate_profile, 2, 80, 10, 200); 687 rate_profile.frame_index_rate_update[3] = kNbrFramesLong + 1; 688 rate_profile.num_frames = kNbrFramesLong; 689 // Codec/network settings. 690 CodecConfigPars process_settings; 691 SetCodecParameters(&process_settings, 0.0f, -1, 1, false, true, true, false); 692 // Metrics for expected quality. 693 QualityMetrics quality_metrics; 694 SetQualityMetrics(&quality_metrics, 31.0, 22.0, 0.80, 0.65); 695 // Metrics for rate control. 696 RateControlMetrics rc_metrics[3]; 697 SetRateControlMetrics(rc_metrics, 0, 40, 20, 75, 15, 60, 0); 698 SetRateControlMetrics(rc_metrics, 1, 10, 0, 25, 10, 35, 0); 699 SetRateControlMetrics(rc_metrics, 2, 0, 0, 20, 10, 15, 0); 700 ProcessFramesAndVerify(quality_metrics, 701 rate_profile, 702 process_settings, 703 rc_metrics); 704 } 705 706 // Run with no packet loss, at low bitrate. During this time we should've 707 // resized once. 708 TEST_F(VideoProcessorIntegrationTest, 709 DISABLED_ON_ANDROID(ProcessNoLossSpatialResizeFrameDrop)) { 710 config_.networking_config.packet_loss_probability = 0; 711 // Bitrate and frame rate profile. 712 RateProfile rate_profile; 713 SetRateProfilePars(&rate_profile, 0, 50, 30, 0); 714 rate_profile.frame_index_rate_update[1] = kNbrFramesLong + 1; 715 rate_profile.num_frames = kNbrFramesLong; 716 // Codec/network settings. 717 CodecConfigPars process_settings; 718 SetCodecParameters( 719 &process_settings, 0.0f, kNbrFramesLong, 1, false, true, true, true); 720 // Metrics for expected quality. 721 QualityMetrics quality_metrics; 722 SetQualityMetrics(&quality_metrics, 25.0, 15.0, 0.70, 0.40); 723 // Metrics for rate control. 724 RateControlMetrics rc_metrics[1]; 725 SetRateControlMetrics(rc_metrics, 0, 160, 60, 120, 20, 70, 1); 726 ProcessFramesAndVerify(quality_metrics, 727 rate_profile, 728 process_settings, 729 rc_metrics); 730 } 731 732 // Run with no packet loss, with 3 temporal layers, with a rate update in the 733 // middle of the sequence. The max values for the frame size mismatch and 734 // encoding rate mismatch are applied to each layer. 735 // No dropped frames in this test, and internal spatial resizer is off. 736 // One key frame (first frame only) in sequence, so no spatial resizing. 737 TEST_F(VideoProcessorIntegrationTest, 738 DISABLED_ON_ANDROID(ProcessNoLossTemporalLayers)) { 739 config_.networking_config.packet_loss_probability = 0; 740 // Bitrate and frame rate profile. 741 RateProfile rate_profile; 742 SetRateProfilePars(&rate_profile, 0, 200, 30, 0); 743 SetRateProfilePars(&rate_profile, 1, 400, 30, 150); 744 rate_profile.frame_index_rate_update[2] = kNbrFramesLong + 1; 745 rate_profile.num_frames = kNbrFramesLong; 746 // Codec/network settings. 747 CodecConfigPars process_settings; 748 SetCodecParameters(&process_settings, 0.0f, -1, 3, false, true, true, false); 749 // Metrics for expected quality. 750 QualityMetrics quality_metrics; 751 SetQualityMetrics(&quality_metrics, 32.5, 30.0, 0.85, 0.80); 752 // Metrics for rate control. 753 RateControlMetrics rc_metrics[2]; 754 SetRateControlMetrics(rc_metrics, 0, 0, 20, 30, 10, 10, 0); 755 SetRateControlMetrics(rc_metrics, 1, 0, 0, 30, 15, 10, 0); 756 ProcessFramesAndVerify(quality_metrics, 757 rate_profile, 758 process_settings, 759 rc_metrics); 760 } 761 } // namespace webrtc 762
