1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include <math.h> 12 13 #include "testing/gtest/include/gtest/gtest.h" 14 15 #include "webrtc/modules/video_coding/include/video_codec_interface.h" 16 #include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h" 17 #include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" 18 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" 19 #include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" 20 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" 21 #include "webrtc/modules/video_coding/include/video_coding.h" 22 #include "webrtc/test/testsupport/fileutils.h" 23 #include "webrtc/test/testsupport/frame_reader.h" 24 #include "webrtc/test/testsupport/frame_writer.h" 25 #include "webrtc/test/testsupport/metrics/video_metrics.h" 26 #include "webrtc/test/testsupport/packet_reader.h" 27 #include "webrtc/typedefs.h" 28 29 namespace webrtc { 30 31 // Maximum number of rate updates (i.e., calls to encoder to change bitrate 32 // and/or frame rate) for the current tests. 33 const int kMaxNumRateUpdates = 3; 34 35 const int kPercTargetvsActualMismatch = 20; 36 const int kBaseKeyFrameInterval = 3000; 37 38 // Codec and network settings. 39 struct CodecConfigPars { 40 VideoCodecType codec_type; 41 float packet_loss; 42 int num_temporal_layers; 43 int key_frame_interval; 44 bool error_concealment_on; 45 bool denoising_on; 46 bool frame_dropper_on; 47 bool spatial_resize_on; 48 }; 49 50 // Quality metrics. 51 struct QualityMetrics { 52 double minimum_avg_psnr; 53 double minimum_min_psnr; 54 double minimum_avg_ssim; 55 double minimum_min_ssim; 56 }; 57 58 // The sequence of bitrate and frame rate changes for the encoder, the frame 59 // number where the changes are made, and the total number of frames for the 60 // test. 61 struct RateProfile { 62 int target_bit_rate[kMaxNumRateUpdates]; 63 int input_frame_rate[kMaxNumRateUpdates]; 64 int frame_index_rate_update[kMaxNumRateUpdates + 1]; 65 int num_frames; 66 }; 67 68 // Metrics for the rate control. The rate mismatch metrics are defined as 69 // percentages.|max_time_hit_target| is defined as number of frames, after a 70 // rate update is made to the encoder, for the encoder to reach within 71 // |kPercTargetvsActualMismatch| of new target rate. The metrics are defined for 72 // each rate update sequence. 73 struct RateControlMetrics { 74 int max_num_dropped_frames; 75 int max_key_frame_size_mismatch; 76 int max_delta_frame_size_mismatch; 77 int max_encoding_rate_mismatch; 78 int max_time_hit_target; 79 int num_spatial_resizes; 80 int num_key_frames; 81 }; 82 83 // Sequence used is foreman (CIF): may be better to use VGA for resize test. 84 const int kCIFWidth = 352; 85 const int kCIFHeight = 288; 86 const int kNbrFramesShort = 100; // Some tests are run for shorter sequence. 87 const int kNbrFramesLong = 299; 88 89 // Parameters from VP8 wrapper, which control target size of key frames. 90 const float kInitialBufferSize = 0.5f; 91 const float kOptimalBufferSize = 0.6f; 92 const float kScaleKeyFrameSize = 0.5f; 93 94 // Integration test for video processor. Encodes+decodes a clip and 95 // writes it to the output directory. After completion, quality metrics 96 // (PSNR and SSIM) and rate control metrics are computed to verify that the 97 // quality and encoder response is acceptable. The rate control tests allow us 98 // to verify the behavior for changing bitrate, changing frame rate, frame 99 // dropping/spatial resize, and temporal layers. The limits for the rate 100 // control metrics are set to be fairly conservative, so failure should only 101 // happen when some significant regression or breakdown occurs. 102 class VideoProcessorIntegrationTest : public testing::Test { 103 protected: 104 VideoEncoder* encoder_; 105 VideoDecoder* decoder_; 106 webrtc::test::FrameReader* frame_reader_; 107 webrtc::test::FrameWriter* frame_writer_; 108 webrtc::test::PacketReader packet_reader_; 109 webrtc::test::PacketManipulator* packet_manipulator_; 110 webrtc::test::Stats stats_; 111 webrtc::test::TestConfig config_; 112 VideoCodec codec_settings_; 113 webrtc::test::VideoProcessor* processor_; 114 115 // Quantities defined/updated for every encoder rate update. 116 // Some quantities defined per temporal layer (at most 3 layers in this test). 117 int num_frames_per_update_[3]; 118 float sum_frame_size_mismatch_[3]; 119 float sum_encoded_frame_size_[3]; 120 float encoding_bitrate_[3]; 121 float per_frame_bandwidth_[3]; 122 float bit_rate_layer_[3]; 123 float frame_rate_layer_[3]; 124 int num_frames_total_; 125 float sum_encoded_frame_size_total_; 126 float encoding_bitrate_total_; 127 float perc_encoding_rate_mismatch_; 128 int num_frames_to_hit_target_; 129 bool encoding_rate_within_target_; 130 int bit_rate_; 131 int frame_rate_; 132 int layer_; 133 float target_size_key_frame_initial_; 134 float target_size_key_frame_; 135 float sum_key_frame_size_mismatch_; 136 int num_key_frames_; 137 float start_bitrate_; 138 139 // Codec and network settings. 140 VideoCodecType codec_type_; 141 float packet_loss_; 142 int num_temporal_layers_; 143 int key_frame_interval_; 144 bool error_concealment_on_; 145 bool denoising_on_; 146 bool frame_dropper_on_; 147 bool spatial_resize_on_; 148 149 VideoProcessorIntegrationTest() {} 150 virtual ~VideoProcessorIntegrationTest() {} 151 152 void SetUpCodecConfig() { 153 if (codec_type_ == kVideoCodecVP8) { 154 encoder_ = VP8Encoder::Create(); 155 decoder_ = VP8Decoder::Create(); 156 VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_); 157 } else if (codec_type_ == kVideoCodecVP9) { 158 encoder_ = VP9Encoder::Create(); 159 decoder_ = VP9Decoder::Create(); 160 VideoCodingModule::Codec(kVideoCodecVP9, &codec_settings_); 161 } 162 163 // CIF is currently used for all tests below. 164 // Setup the TestConfig struct for processing of a clip in CIF resolution. 165 config_.input_filename = webrtc::test::ResourcePath("foreman_cif", "yuv"); 166 167 // Generate an output filename in a safe way. 168 config_.output_filename = webrtc::test::TempFilename( 169 webrtc::test::OutputPath(), "videoprocessor_integrationtest"); 170 config_.frame_length_in_bytes = 171 CalcBufferSize(kI420, kCIFWidth, kCIFHeight); 172 config_.verbose = false; 173 // Only allow encoder/decoder to use single core, for predictability. 174 config_.use_single_core = true; 175 // Key frame interval and packet loss are set for each test. 176 config_.keyframe_interval = key_frame_interval_; 177 config_.networking_config.packet_loss_probability = packet_loss_; 178 179 // Configure codec settings. 180 config_.codec_settings = &codec_settings_; 181 config_.codec_settings->startBitrate = start_bitrate_; 182 config_.codec_settings->width = kCIFWidth; 183 config_.codec_settings->height = kCIFHeight; 184 185 // These features may be set depending on the test. 186 switch (config_.codec_settings->codecType) { 187 case kVideoCodecVP8: 188 config_.codec_settings->codecSpecific.VP8.errorConcealmentOn = 189 error_concealment_on_; 190 config_.codec_settings->codecSpecific.VP8.denoisingOn = denoising_on_; 191 config_.codec_settings->codecSpecific.VP8.numberOfTemporalLayers = 192 num_temporal_layers_; 193 config_.codec_settings->codecSpecific.VP8.frameDroppingOn = 194 frame_dropper_on_; 195 config_.codec_settings->codecSpecific.VP8.automaticResizeOn = 196 spatial_resize_on_; 197 config_.codec_settings->codecSpecific.VP8.keyFrameInterval = 198 kBaseKeyFrameInterval; 199 break; 200 case kVideoCodecVP9: 201 config_.codec_settings->codecSpecific.VP9.denoisingOn = denoising_on_; 202 config_.codec_settings->codecSpecific.VP9.numberOfTemporalLayers = 203 num_temporal_layers_; 204 config_.codec_settings->codecSpecific.VP9.frameDroppingOn = 205 frame_dropper_on_; 206 config_.codec_settings->codecSpecific.VP9.automaticResizeOn = 207 spatial_resize_on_; 208 config_.codec_settings->codecSpecific.VP9.keyFrameInterval = 209 kBaseKeyFrameInterval; 210 break; 211 default: 212 assert(false); 213 break; 214 } 215 frame_reader_ = new webrtc::test::FrameReaderImpl( 216 config_.input_filename, config_.frame_length_in_bytes); 217 frame_writer_ = new webrtc::test::FrameWriterImpl( 218 config_.output_filename, config_.frame_length_in_bytes); 219 ASSERT_TRUE(frame_reader_->Init()); 220 ASSERT_TRUE(frame_writer_->Init()); 221 222 packet_manipulator_ = new webrtc::test::PacketManipulatorImpl( 223 &packet_reader_, config_.networking_config, config_.verbose); 224 processor_ = new webrtc::test::VideoProcessorImpl( 225 encoder_, decoder_, frame_reader_, frame_writer_, packet_manipulator_, 226 config_, &stats_); 227 ASSERT_TRUE(processor_->Init()); 228 } 229 230 // Reset quantities after each encoder update, update the target 231 // per-frame bandwidth. 232 void ResetRateControlMetrics(int num_frames) { 233 for (int i = 0; i < num_temporal_layers_; i++) { 234 num_frames_per_update_[i] = 0; 235 sum_frame_size_mismatch_[i] = 0.0f; 236 sum_encoded_frame_size_[i] = 0.0f; 237 encoding_bitrate_[i] = 0.0f; 238 // Update layer per-frame-bandwidth. 239 per_frame_bandwidth_[i] = static_cast<float>(bit_rate_layer_[i]) / 240 static_cast<float>(frame_rate_layer_[i]); 241 } 242 // Set maximum size of key frames, following setting in the VP8 wrapper. 243 float max_key_size = kScaleKeyFrameSize * kOptimalBufferSize * frame_rate_; 244 // We don't know exact target size of the key frames (except for first one), 245 // but the minimum in libvpx is ~|3 * per_frame_bandwidth| and maximum is 246 // set by |max_key_size_ * per_frame_bandwidth|. Take middle point/average 247 // as reference for mismatch. Note key frames always correspond to base 248 // layer frame in this test. 249 target_size_key_frame_ = 0.5 * (3 + max_key_size) * per_frame_bandwidth_[0]; 250 num_frames_total_ = 0; 251 sum_encoded_frame_size_total_ = 0.0f; 252 encoding_bitrate_total_ = 0.0f; 253 perc_encoding_rate_mismatch_ = 0.0f; 254 num_frames_to_hit_target_ = num_frames; 255 encoding_rate_within_target_ = false; 256 sum_key_frame_size_mismatch_ = 0.0; 257 num_key_frames_ = 0; 258 } 259 260 // For every encoded frame, update the rate control metrics. 261 void UpdateRateControlMetrics(int frame_num, FrameType frame_type) { 262 float encoded_size_kbits = processor_->EncodedFrameSize() * 8.0f / 1000.0f; 263 // Update layer data. 264 // Update rate mismatch relative to per-frame bandwidth for delta frames. 265 if (frame_type == kVideoFrameDelta) { 266 // TODO(marpan): Should we count dropped (zero size) frames in mismatch? 267 sum_frame_size_mismatch_[layer_] += 268 fabs(encoded_size_kbits - per_frame_bandwidth_[layer_]) / 269 per_frame_bandwidth_[layer_]; 270 } else { 271 float target_size = (frame_num == 1) ? target_size_key_frame_initial_ 272 : target_size_key_frame_; 273 sum_key_frame_size_mismatch_ += 274 fabs(encoded_size_kbits - target_size) / target_size; 275 num_key_frames_ += 1; 276 } 277 sum_encoded_frame_size_[layer_] += encoded_size_kbits; 278 // Encoding bitrate per layer: from the start of the update/run to the 279 // current frame. 280 encoding_bitrate_[layer_] = sum_encoded_frame_size_[layer_] * 281 frame_rate_layer_[layer_] / 282 num_frames_per_update_[layer_]; 283 // Total encoding rate: from the start of the update/run to current frame. 284 sum_encoded_frame_size_total_ += encoded_size_kbits; 285 encoding_bitrate_total_ = 286 sum_encoded_frame_size_total_ * frame_rate_ / num_frames_total_; 287 perc_encoding_rate_mismatch_ = 288 100 * fabs(encoding_bitrate_total_ - bit_rate_) / bit_rate_; 289 if (perc_encoding_rate_mismatch_ < kPercTargetvsActualMismatch && 290 !encoding_rate_within_target_) { 291 num_frames_to_hit_target_ = num_frames_total_; 292 encoding_rate_within_target_ = true; 293 } 294 } 295 296 // Verify expected behavior of rate control and print out data. 297 void VerifyRateControl(int update_index, 298 int max_key_frame_size_mismatch, 299 int max_delta_frame_size_mismatch, 300 int max_encoding_rate_mismatch, 301 int max_time_hit_target, 302 int max_num_dropped_frames, 303 int num_spatial_resizes, 304 int num_key_frames) { 305 int num_dropped_frames = processor_->NumberDroppedFrames(); 306 int num_resize_actions = processor_->NumberSpatialResizes(); 307 printf( 308 "For update #: %d,\n " 309 " Target Bitrate: %d,\n" 310 " Encoding bitrate: %f,\n" 311 " Frame rate: %d \n", 312 update_index, bit_rate_, encoding_bitrate_total_, frame_rate_); 313 printf( 314 " Number of frames to approach target rate = %d, \n" 315 " Number of dropped frames = %d, \n" 316 " Number of spatial resizes = %d, \n", 317 num_frames_to_hit_target_, num_dropped_frames, num_resize_actions); 318 EXPECT_LE(perc_encoding_rate_mismatch_, max_encoding_rate_mismatch); 319 if (num_key_frames_ > 0) { 320 int perc_key_frame_size_mismatch = 321 100 * sum_key_frame_size_mismatch_ / num_key_frames_; 322 printf( 323 " Number of Key frames: %d \n" 324 " Key frame rate mismatch: %d \n", 325 num_key_frames_, perc_key_frame_size_mismatch); 326 EXPECT_LE(perc_key_frame_size_mismatch, max_key_frame_size_mismatch); 327 } 328 printf("\n"); 329 printf("Rates statistics for Layer data \n"); 330 for (int i = 0; i < num_temporal_layers_; i++) { 331 printf("Layer #%d \n", i); 332 int perc_frame_size_mismatch = 333 100 * sum_frame_size_mismatch_[i] / num_frames_per_update_[i]; 334 int perc_encoding_rate_mismatch = 335 100 * fabs(encoding_bitrate_[i] - bit_rate_layer_[i]) / 336 bit_rate_layer_[i]; 337 printf( 338 " Target Layer Bit rate: %f \n" 339 " Layer frame rate: %f, \n" 340 " Layer per frame bandwidth: %f, \n" 341 " Layer Encoding bit rate: %f, \n" 342 " Layer Percent frame size mismatch: %d, \n" 343 " Layer Percent encoding rate mismatch = %d, \n" 344 " Number of frame processed per layer = %d \n", 345 bit_rate_layer_[i], frame_rate_layer_[i], per_frame_bandwidth_[i], 346 encoding_bitrate_[i], perc_frame_size_mismatch, 347 perc_encoding_rate_mismatch, num_frames_per_update_[i]); 348 EXPECT_LE(perc_frame_size_mismatch, max_delta_frame_size_mismatch); 349 EXPECT_LE(perc_encoding_rate_mismatch, max_encoding_rate_mismatch); 350 } 351 printf("\n"); 352 EXPECT_LE(num_frames_to_hit_target_, max_time_hit_target); 353 EXPECT_LE(num_dropped_frames, max_num_dropped_frames); 354 EXPECT_EQ(num_resize_actions, num_spatial_resizes); 355 EXPECT_EQ(num_key_frames_, num_key_frames); 356 } 357 358 // Layer index corresponding to frame number, for up to 3 layers. 359 void LayerIndexForFrame(int frame_number) { 360 if (num_temporal_layers_ == 1) { 361 layer_ = 0; 362 } else if (num_temporal_layers_ == 2) { 363 // layer 0: 0 2 4 ... 364 // layer 1: 1 3 365 if (frame_number % 2 == 0) { 366 layer_ = 0; 367 } else { 368 layer_ = 1; 369 } 370 } else if (num_temporal_layers_ == 3) { 371 // layer 0: 0 4 8 ... 372 // layer 1: 2 6 373 // layer 2: 1 3 5 7 374 if (frame_number % 4 == 0) { 375 layer_ = 0; 376 } else if ((frame_number + 2) % 4 == 0) { 377 layer_ = 1; 378 } else if ((frame_number + 1) % 2 == 0) { 379 layer_ = 2; 380 } 381 } else { 382 assert(false); // Only up to 3 layers. 383 } 384 } 385 386 // Set the bitrate and frame rate per layer, for up to 3 layers. 387 void SetLayerRates() { 388 assert(num_temporal_layers_ <= 3); 389 for (int i = 0; i < num_temporal_layers_; i++) { 390 float bit_rate_ratio = 391 kVp8LayerRateAlloction[num_temporal_layers_ - 1][i]; 392 if (i > 0) { 393 float bit_rate_delta_ratio = 394 kVp8LayerRateAlloction[num_temporal_layers_ - 1][i] - 395 kVp8LayerRateAlloction[num_temporal_layers_ - 1][i - 1]; 396 bit_rate_layer_[i] = bit_rate_ * bit_rate_delta_ratio; 397 } else { 398 bit_rate_layer_[i] = bit_rate_ * bit_rate_ratio; 399 } 400 frame_rate_layer_[i] = 401 frame_rate_ / static_cast<float>(1 << (num_temporal_layers_ - 1)); 402 } 403 if (num_temporal_layers_ == 3) { 404 frame_rate_layer_[2] = frame_rate_ / 2.0f; 405 } 406 } 407 408 void TearDown() { 409 delete processor_; 410 delete packet_manipulator_; 411 delete frame_writer_; 412 delete frame_reader_; 413 delete decoder_; 414 delete encoder_; 415 } 416 417 // Processes all frames in the clip and verifies the result. 418 void ProcessFramesAndVerify(QualityMetrics quality_metrics, 419 RateProfile rate_profile, 420 CodecConfigPars process, 421 RateControlMetrics* rc_metrics) { 422 // Codec/config settings. 423 codec_type_ = process.codec_type; 424 start_bitrate_ = rate_profile.target_bit_rate[0]; 425 packet_loss_ = process.packet_loss; 426 key_frame_interval_ = process.key_frame_interval; 427 num_temporal_layers_ = process.num_temporal_layers; 428 error_concealment_on_ = process.error_concealment_on; 429 denoising_on_ = process.denoising_on; 430 frame_dropper_on_ = process.frame_dropper_on; 431 spatial_resize_on_ = process.spatial_resize_on; 432 SetUpCodecConfig(); 433 // Update the layers and the codec with the initial rates. 434 bit_rate_ = rate_profile.target_bit_rate[0]; 435 frame_rate_ = rate_profile.input_frame_rate[0]; 436 SetLayerRates(); 437 // Set the initial target size for key frame. 438 target_size_key_frame_initial_ = 439 0.5 * kInitialBufferSize * bit_rate_layer_[0]; 440 processor_->SetRates(bit_rate_, frame_rate_); 441 // Process each frame, up to |num_frames|. 442 int num_frames = rate_profile.num_frames; 443 int update_index = 0; 444 ResetRateControlMetrics( 445 rate_profile.frame_index_rate_update[update_index + 1]); 446 int frame_number = 0; 447 FrameType frame_type = kVideoFrameDelta; 448 while (processor_->ProcessFrame(frame_number) && 449 frame_number < num_frames) { 450 // Get the layer index for the frame |frame_number|. 451 LayerIndexForFrame(frame_number); 452 // Get the frame_type. 453 frame_type = processor_->EncodedFrameType(); 454 // Counter for whole sequence run. 455 ++frame_number; 456 // Counters for each rate update. 457 ++num_frames_per_update_[layer_]; 458 ++num_frames_total_; 459 UpdateRateControlMetrics(frame_number, frame_type); 460 // If we hit another/next update, verify stats for current state and 461 // update layers and codec with new rates. 462 if (frame_number == 463 rate_profile.frame_index_rate_update[update_index + 1]) { 464 VerifyRateControl( 465 update_index, rc_metrics[update_index].max_key_frame_size_mismatch, 466 rc_metrics[update_index].max_delta_frame_size_mismatch, 467 rc_metrics[update_index].max_encoding_rate_mismatch, 468 rc_metrics[update_index].max_time_hit_target, 469 rc_metrics[update_index].max_num_dropped_frames, 470 rc_metrics[update_index].num_spatial_resizes, 471 rc_metrics[update_index].num_key_frames); 472 // Update layer rates and the codec with new rates. 473 ++update_index; 474 bit_rate_ = rate_profile.target_bit_rate[update_index]; 475 frame_rate_ = rate_profile.input_frame_rate[update_index]; 476 SetLayerRates(); 477 ResetRateControlMetrics( 478 rate_profile.frame_index_rate_update[update_index + 1]); 479 processor_->SetRates(bit_rate_, frame_rate_); 480 } 481 } 482 VerifyRateControl(update_index, 483 rc_metrics[update_index].max_key_frame_size_mismatch, 484 rc_metrics[update_index].max_delta_frame_size_mismatch, 485 rc_metrics[update_index].max_encoding_rate_mismatch, 486 rc_metrics[update_index].max_time_hit_target, 487 rc_metrics[update_index].max_num_dropped_frames, 488 rc_metrics[update_index].num_spatial_resizes, 489 rc_metrics[update_index].num_key_frames); 490 EXPECT_EQ(num_frames, frame_number); 491 EXPECT_EQ(num_frames + 1, static_cast<int>(stats_.stats_.size())); 492 493 // Release encoder and decoder to make sure they have finished processing: 494 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); 495 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release()); 496 // Close the files before we start using them for SSIM/PSNR calculations. 497 frame_reader_->Close(); 498 frame_writer_->Close(); 499 500 // TODO(marpan): should compute these quality metrics per SetRates update. 501 webrtc::test::QualityMetricsResult psnr_result, ssim_result; 502 EXPECT_EQ( 503 0, webrtc::test::I420MetricsFromFiles( 504 config_.input_filename.c_str(), config_.output_filename.c_str(), 505 config_.codec_settings->width, config_.codec_settings->height, 506 &psnr_result, &ssim_result)); 507 printf("PSNR avg: %f, min: %f SSIM avg: %f, min: %f\n", 508 psnr_result.average, psnr_result.min, ssim_result.average, 509 ssim_result.min); 510 stats_.PrintSummary(); 511 EXPECT_GT(psnr_result.average, quality_metrics.minimum_avg_psnr); 512 EXPECT_GT(psnr_result.min, quality_metrics.minimum_min_psnr); 513 EXPECT_GT(ssim_result.average, quality_metrics.minimum_avg_ssim); 514 EXPECT_GT(ssim_result.min, quality_metrics.minimum_min_ssim); 515 if (!remove(config_.output_filename.c_str())) { 516 fprintf(stderr, "Failed to remove temporary file!"); 517 } 518 } 519 }; 520 521 void SetRateProfilePars(RateProfile* rate_profile, 522 int update_index, 523 int bit_rate, 524 int frame_rate, 525 int frame_index_rate_update) { 526 rate_profile->target_bit_rate[update_index] = bit_rate; 527 rate_profile->input_frame_rate[update_index] = frame_rate; 528 rate_profile->frame_index_rate_update[update_index] = frame_index_rate_update; 529 } 530 531 void SetCodecParameters(CodecConfigPars* process_settings, 532 VideoCodecType codec_type, 533 float packet_loss, 534 int key_frame_interval, 535 int num_temporal_layers, 536 bool error_concealment_on, 537 bool denoising_on, 538 bool frame_dropper_on, 539 bool spatial_resize_on) { 540 process_settings->codec_type = codec_type; 541 process_settings->packet_loss = packet_loss; 542 process_settings->key_frame_interval = key_frame_interval; 543 process_settings->num_temporal_layers = num_temporal_layers, 544 process_settings->error_concealment_on = error_concealment_on; 545 process_settings->denoising_on = denoising_on; 546 process_settings->frame_dropper_on = frame_dropper_on; 547 process_settings->spatial_resize_on = spatial_resize_on; 548 } 549 550 void SetQualityMetrics(QualityMetrics* quality_metrics, 551 double minimum_avg_psnr, 552 double minimum_min_psnr, 553 double minimum_avg_ssim, 554 double minimum_min_ssim) { 555 quality_metrics->minimum_avg_psnr = minimum_avg_psnr; 556 quality_metrics->minimum_min_psnr = minimum_min_psnr; 557 quality_metrics->minimum_avg_ssim = minimum_avg_ssim; 558 quality_metrics->minimum_min_ssim = minimum_min_ssim; 559 } 560 561 void SetRateControlMetrics(RateControlMetrics* rc_metrics, 562 int update_index, 563 int max_num_dropped_frames, 564 int max_key_frame_size_mismatch, 565 int max_delta_frame_size_mismatch, 566 int max_encoding_rate_mismatch, 567 int max_time_hit_target, 568 int num_spatial_resizes, 569 int num_key_frames) { 570 rc_metrics[update_index].max_num_dropped_frames = max_num_dropped_frames; 571 rc_metrics[update_index].max_key_frame_size_mismatch = 572 max_key_frame_size_mismatch; 573 rc_metrics[update_index].max_delta_frame_size_mismatch = 574 max_delta_frame_size_mismatch; 575 rc_metrics[update_index].max_encoding_rate_mismatch = 576 max_encoding_rate_mismatch; 577 rc_metrics[update_index].max_time_hit_target = max_time_hit_target; 578 rc_metrics[update_index].num_spatial_resizes = num_spatial_resizes; 579 rc_metrics[update_index].num_key_frames = num_key_frames; 580 } 581 582 // VP9: Run with no packet loss and fixed bitrate. Quality should be very high. 583 // One key frame (first frame only) in sequence. Setting |key_frame_interval| 584 // to -1 below means no periodic key frames in test. 585 TEST_F(VideoProcessorIntegrationTest, Process0PercentPacketLossVP9) { 586 // Bitrate and frame rate profile. 587 RateProfile rate_profile; 588 SetRateProfilePars(&rate_profile, 0, 500, 30, 0); 589 rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; 590 rate_profile.num_frames = kNbrFramesShort; 591 // Codec/network settings. 592 CodecConfigPars process_settings; 593 SetCodecParameters(&process_settings, kVideoCodecVP9, 0.0f, -1, 1, false, 594 false, true, false); 595 // Metrics for expected quality. 596 QualityMetrics quality_metrics; 597 SetQualityMetrics(&quality_metrics, 37.0, 36.0, 0.93, 0.92); 598 // Metrics for rate control. 599 RateControlMetrics rc_metrics[1]; 600 SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 20, 0, 1); 601 ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, 602 rc_metrics); 603 } 604 605 // VP9: Run with 5% packet loss and fixed bitrate. Quality should be a bit 606 // lower. One key frame (first frame only) in sequence. 607 TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLossVP9) { 608 // Bitrate and frame rate profile. 609 RateProfile rate_profile; 610 SetRateProfilePars(&rate_profile, 0, 500, 30, 0); 611 rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; 612 rate_profile.num_frames = kNbrFramesShort; 613 // Codec/network settings. 614 CodecConfigPars process_settings; 615 SetCodecParameters(&process_settings, kVideoCodecVP9, 0.05f, -1, 1, false, 616 false, true, false); 617 // Metrics for expected quality. 618 QualityMetrics quality_metrics; 619 SetQualityMetrics(&quality_metrics, 17.0, 14.0, 0.45, 0.36); 620 // Metrics for rate control. 621 RateControlMetrics rc_metrics[1]; 622 SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 20, 0, 1); 623 ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, 624 rc_metrics); 625 } 626 627 // VP9: Run with no packet loss, with varying bitrate (3 rate updates): 628 // low to high to medium. Check that quality and encoder response to the new 629 // target rate/per-frame bandwidth (for each rate update) is within limits. 630 // One key frame (first frame only) in sequence. 631 TEST_F(VideoProcessorIntegrationTest, ProcessNoLossChangeBitRateVP9) { 632 // Bitrate and frame rate profile. 633 RateProfile rate_profile; 634 SetRateProfilePars(&rate_profile, 0, 200, 30, 0); 635 SetRateProfilePars(&rate_profile, 1, 700, 30, 100); 636 SetRateProfilePars(&rate_profile, 2, 500, 30, 200); 637 rate_profile.frame_index_rate_update[3] = kNbrFramesLong + 1; 638 rate_profile.num_frames = kNbrFramesLong; 639 // Codec/network settings. 640 CodecConfigPars process_settings; 641 SetCodecParameters(&process_settings, kVideoCodecVP9, 0.0f, -1, 1, false, 642 false, true, false); 643 // Metrics for expected quality. 644 QualityMetrics quality_metrics; 645 SetQualityMetrics(&quality_metrics, 35.7, 30.0, 0.90, 0.85); 646 // Metrics for rate control. 647 RateControlMetrics rc_metrics[3]; 648 SetRateControlMetrics(rc_metrics, 0, 0, 30, 20, 20, 30, 0, 1); 649 SetRateControlMetrics(rc_metrics, 1, 2, 0, 20, 20, 60, 0, 0); 650 SetRateControlMetrics(rc_metrics, 2, 0, 0, 25, 20, 40, 0, 0); 651 ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, 652 rc_metrics); 653 } 654 655 // VP9: Run with no packet loss, with an update (decrease) in frame rate. 656 // Lower frame rate means higher per-frame-bandwidth, so easier to encode. 657 // At the low bitrate in this test, this means better rate control after the 658 // update(s) to lower frame rate. So expect less frame drops, and max values 659 // for the rate control metrics can be lower. One key frame (first frame only). 660 // Note: quality after update should be higher but we currently compute quality 661 // metrics averaged over whole sequence run. 662 TEST_F(VideoProcessorIntegrationTest, 663 ProcessNoLossChangeFrameRateFrameDropVP9) { 664 config_.networking_config.packet_loss_probability = 0; 665 // Bitrate and frame rate profile. 666 RateProfile rate_profile; 667 SetRateProfilePars(&rate_profile, 0, 100, 24, 0); 668 SetRateProfilePars(&rate_profile, 1, 100, 15, 100); 669 SetRateProfilePars(&rate_profile, 2, 100, 10, 200); 670 rate_profile.frame_index_rate_update[3] = kNbrFramesLong + 1; 671 rate_profile.num_frames = kNbrFramesLong; 672 // Codec/network settings. 673 CodecConfigPars process_settings; 674 SetCodecParameters(&process_settings, kVideoCodecVP9, 0.0f, -1, 1, false, 675 false, true, false); 676 // Metrics for expected quality. 677 QualityMetrics quality_metrics; 678 SetQualityMetrics(&quality_metrics, 31.5, 18.0, 0.80, 0.44); 679 // Metrics for rate control. 680 RateControlMetrics rc_metrics[3]; 681 SetRateControlMetrics(rc_metrics, 0, 38, 50, 75, 15, 45, 0, 1); 682 SetRateControlMetrics(rc_metrics, 1, 10, 0, 40, 10, 30, 0, 0); 683 SetRateControlMetrics(rc_metrics, 2, 5, 0, 30, 5, 20, 0, 0); 684 ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, 685 rc_metrics); 686 } 687 688 // VP9: Run with no packet loss and denoiser on. One key frame (first frame). 689 TEST_F(VideoProcessorIntegrationTest, ProcessNoLossDenoiserOnVP9) { 690 // Bitrate and frame rate profile. 691 RateProfile rate_profile; 692 SetRateProfilePars(&rate_profile, 0, 500, 30, 0); 693 rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; 694 rate_profile.num_frames = kNbrFramesShort; 695 // Codec/network settings. 696 CodecConfigPars process_settings; 697 SetCodecParameters(&process_settings, kVideoCodecVP9, 0.0f, -1, 1, false, 698 true, true, false); 699 // Metrics for expected quality. 700 QualityMetrics quality_metrics; 701 SetQualityMetrics(&quality_metrics, 36.8, 35.8, 0.92, 0.91); 702 // Metrics for rate control. 703 RateControlMetrics rc_metrics[1]; 704 SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 20, 0, 1); 705 ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, 706 rc_metrics); 707 } 708 709 // Run with no packet loss, at low bitrate. 710 // spatial_resize is on, for this low bitrate expect one resize in sequence. 711 // Resize happens on delta frame. Expect only one key frame (first frame). 712 TEST_F(VideoProcessorIntegrationTest, ProcessNoLossSpatialResizeFrameDropVP9) { 713 config_.networking_config.packet_loss_probability = 0; 714 // Bitrate and frame rate profile. 715 RateProfile rate_profile; 716 SetRateProfilePars(&rate_profile, 0, 50, 30, 0); 717 rate_profile.frame_index_rate_update[1] = kNbrFramesLong + 1; 718 rate_profile.num_frames = kNbrFramesLong; 719 // Codec/network settings. 720 CodecConfigPars process_settings; 721 SetCodecParameters(&process_settings, kVideoCodecVP9, 0.0f, -1, 1, false, 722 false, true, true); 723 // Metrics for expected quality. 724 QualityMetrics quality_metrics; 725 SetQualityMetrics(&quality_metrics, 24.0, 13.0, 0.65, 0.37); 726 // Metrics for rate control. 727 RateControlMetrics rc_metrics[1]; 728 SetRateControlMetrics(rc_metrics, 0, 228, 70, 160, 15, 80, 1, 1); 729 ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, 730 rc_metrics); 731 } 732 733 // TODO(marpan): Add temporal layer test for VP9, once changes are in 734 // vp9 wrapper for this. 735 736 // VP8: Run with no packet loss and fixed bitrate. Quality should be very high. 737 // One key frame (first frame only) in sequence. Setting |key_frame_interval| 738 // to -1 below means no periodic key frames in test. 739 TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) { 740 // Bitrate and frame rate profile. 741 RateProfile rate_profile; 742 SetRateProfilePars(&rate_profile, 0, 500, 30, 0); 743 rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; 744 rate_profile.num_frames = kNbrFramesShort; 745 // Codec/network settings. 746 CodecConfigPars process_settings; 747 SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, -1, 1, false, 748 true, true, false); 749 // Metrics for expected quality. 750 QualityMetrics quality_metrics; 751 SetQualityMetrics(&quality_metrics, 34.95, 33.0, 0.90, 0.89); 752 // Metrics for rate control. 753 RateControlMetrics rc_metrics[1]; 754 SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 15, 0, 1); 755 ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, 756 rc_metrics); 757 } 758 759 // VP8: Run with 5% packet loss and fixed bitrate. Quality should be a bit 760 // lower. One key frame (first frame only) in sequence. 761 TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) { 762 // Bitrate and frame rate profile. 763 RateProfile rate_profile; 764 SetRateProfilePars(&rate_profile, 0, 500, 30, 0); 765 rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; 766 rate_profile.num_frames = kNbrFramesShort; 767 // Codec/network settings. 768 CodecConfigPars process_settings; 769 SetCodecParameters(&process_settings, kVideoCodecVP8, 0.05f, -1, 1, false, 770 true, true, false); 771 // Metrics for expected quality. 772 QualityMetrics quality_metrics; 773 SetQualityMetrics(&quality_metrics, 20.0, 16.0, 0.60, 0.40); 774 // Metrics for rate control. 775 RateControlMetrics rc_metrics[1]; 776 SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 15, 0, 1); 777 ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, 778 rc_metrics); 779 } 780 781 // VP8: Run with 10% packet loss and fixed bitrate. Quality should be lower. 782 // One key frame (first frame only) in sequence. 783 TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) { 784 // Bitrate and frame rate profile. 785 RateProfile rate_profile; 786 SetRateProfilePars(&rate_profile, 0, 500, 30, 0); 787 rate_profile.frame_index_rate_update[1] = kNbrFramesShort + 1; 788 rate_profile.num_frames = kNbrFramesShort; 789 // Codec/network settings. 790 CodecConfigPars process_settings; 791 SetCodecParameters(&process_settings, kVideoCodecVP8, 0.1f, -1, 1, false, 792 true, true, false); 793 // Metrics for expected quality. 794 QualityMetrics quality_metrics; 795 SetQualityMetrics(&quality_metrics, 19.0, 16.0, 0.50, 0.35); 796 // Metrics for rate control. 797 RateControlMetrics rc_metrics[1]; 798 SetRateControlMetrics(rc_metrics, 0, 0, 40, 20, 10, 15, 0, 1); 799 ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, 800 rc_metrics); 801 } 802 803 // The tests below are currently disabled for Android. For ARM, the encoder 804 // uses |cpu_speed| = 12, as opposed to default |cpu_speed| <= 6 for x86, 805 // which leads to significantly different quality. The quality and rate control 806 // settings in the tests below are defined for encoder speed setting 807 // |cpu_speed| <= ~6. A number of settings would need to be significantly 808 // modified for the |cpu_speed| = 12 case. For now, keep the tests below 809 // disabled on Android. Some quality parameter in the above test has been 810 // adjusted to also pass for |cpu_speed| <= 12. 811 812 // VP8: Run with no packet loss, with varying bitrate (3 rate updates): 813 // low to high to medium. Check that quality and encoder response to the new 814 // target rate/per-frame bandwidth (for each rate update) is within limits. 815 // One key frame (first frame only) in sequence. 816 #if defined(WEBRTC_ANDROID) 817 #define MAYBE_ProcessNoLossChangeBitRateVP8 \ 818 DISABLED_ProcessNoLossChangeBitRateVP8 819 #else 820 #define MAYBE_ProcessNoLossChangeBitRateVP8 ProcessNoLossChangeBitRateVP8 821 #endif 822 TEST_F(VideoProcessorIntegrationTest, MAYBE_ProcessNoLossChangeBitRateVP8) { 823 // Bitrate and frame rate profile. 824 RateProfile rate_profile; 825 SetRateProfilePars(&rate_profile, 0, 200, 30, 0); 826 SetRateProfilePars(&rate_profile, 1, 800, 30, 100); 827 SetRateProfilePars(&rate_profile, 2, 500, 30, 200); 828 rate_profile.frame_index_rate_update[3] = kNbrFramesLong + 1; 829 rate_profile.num_frames = kNbrFramesLong; 830 // Codec/network settings. 831 CodecConfigPars process_settings; 832 SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, -1, 1, false, 833 true, true, false); 834 // Metrics for expected quality. 835 QualityMetrics quality_metrics; 836 SetQualityMetrics(&quality_metrics, 34.0, 32.0, 0.85, 0.80); 837 // Metrics for rate control. 838 RateControlMetrics rc_metrics[3]; 839 SetRateControlMetrics(rc_metrics, 0, 0, 45, 20, 10, 15, 0, 1); 840 SetRateControlMetrics(rc_metrics, 1, 0, 0, 25, 20, 10, 0, 0); 841 SetRateControlMetrics(rc_metrics, 2, 0, 0, 25, 15, 10, 0, 0); 842 ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, 843 rc_metrics); 844 } 845 846 // VP8: Run with no packet loss, with an update (decrease) in frame rate. 847 // Lower frame rate means higher per-frame-bandwidth, so easier to encode. 848 // At the bitrate in this test, this means better rate control after the 849 // update(s) to lower frame rate. So expect less frame drops, and max values 850 // for the rate control metrics can be lower. One key frame (first frame only). 851 // Note: quality after update should be higher but we currently compute quality 852 // metrics averaged over whole sequence run. 853 #if defined(WEBRTC_ANDROID) 854 #define MAYBE_ProcessNoLossChangeFrameRateFrameDropVP8 \ 855 DISABLED_ProcessNoLossChangeFrameRateFrameDropVP8 856 #else 857 #define MAYBE_ProcessNoLossChangeFrameRateFrameDropVP8 \ 858 ProcessNoLossChangeFrameRateFrameDropVP8 859 #endif 860 TEST_F(VideoProcessorIntegrationTest, 861 MAYBE_ProcessNoLossChangeFrameRateFrameDropVP8) { 862 config_.networking_config.packet_loss_probability = 0; 863 // Bitrate and frame rate profile. 864 RateProfile rate_profile; 865 SetRateProfilePars(&rate_profile, 0, 80, 24, 0); 866 SetRateProfilePars(&rate_profile, 1, 80, 15, 100); 867 SetRateProfilePars(&rate_profile, 2, 80, 10, 200); 868 rate_profile.frame_index_rate_update[3] = kNbrFramesLong + 1; 869 rate_profile.num_frames = kNbrFramesLong; 870 // Codec/network settings. 871 CodecConfigPars process_settings; 872 SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, -1, 1, false, 873 true, true, false); 874 // Metrics for expected quality. 875 QualityMetrics quality_metrics; 876 SetQualityMetrics(&quality_metrics, 31.0, 22.0, 0.80, 0.65); 877 // Metrics for rate control. 878 RateControlMetrics rc_metrics[3]; 879 SetRateControlMetrics(rc_metrics, 0, 40, 20, 75, 15, 60, 0, 1); 880 SetRateControlMetrics(rc_metrics, 1, 10, 0, 25, 10, 35, 0, 0); 881 SetRateControlMetrics(rc_metrics, 2, 0, 0, 20, 10, 15, 0, 0); 882 ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, 883 rc_metrics); 884 } 885 886 // Run with no packet loss, at low bitrate. During this time we should've 887 // resized once. Expect 2 key frames generated (first and one for resize). 888 #if defined(WEBRTC_ANDROID) 889 #define MAYBE_ProcessNoLossSpatialResizeFrameDropVP8 \ 890 DISABLED_ProcessNoLossSpatialResizeFrameDropVP8 891 #else 892 #define MAYBE_ProcessNoLossSpatialResizeFrameDropVP8 \ 893 ProcessNoLossSpatialResizeFrameDropVP8 894 #endif 895 TEST_F(VideoProcessorIntegrationTest, 896 MAYBE_ProcessNoLossSpatialResizeFrameDropVP8) { 897 config_.networking_config.packet_loss_probability = 0; 898 // Bitrate and frame rate profile. 899 RateProfile rate_profile; 900 SetRateProfilePars(&rate_profile, 0, 50, 30, 0); 901 rate_profile.frame_index_rate_update[1] = kNbrFramesLong + 1; 902 rate_profile.num_frames = kNbrFramesLong; 903 // Codec/network settings. 904 CodecConfigPars process_settings; 905 SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, -1, 1, false, 906 true, true, true); 907 // Metrics for expected quality. 908 QualityMetrics quality_metrics; 909 SetQualityMetrics(&quality_metrics, 25.0, 15.0, 0.70, 0.40); 910 // Metrics for rate control. 911 RateControlMetrics rc_metrics[1]; 912 SetRateControlMetrics(rc_metrics, 0, 160, 60, 120, 20, 70, 1, 2); 913 ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, 914 rc_metrics); 915 } 916 917 // VP8: Run with no packet loss, with 3 temporal layers, with a rate update in 918 // the middle of the sequence. The max values for the frame size mismatch and 919 // encoding rate mismatch are applied to each layer. 920 // No dropped frames in this test, and internal spatial resizer is off. 921 // One key frame (first frame only) in sequence, so no spatial resizing. 922 #if defined(WEBRTC_ANDROID) 923 #define MAYBE_ProcessNoLossTemporalLayersVP8 \ 924 DISABLED_ProcessNoLossTemporalLayersVP8 925 #else 926 #define MAYBE_ProcessNoLossTemporalLayersVP8 ProcessNoLossTemporalLayersVP8 927 #endif 928 TEST_F(VideoProcessorIntegrationTest, MAYBE_ProcessNoLossTemporalLayersVP8) { 929 config_.networking_config.packet_loss_probability = 0; 930 // Bitrate and frame rate profile. 931 RateProfile rate_profile; 932 SetRateProfilePars(&rate_profile, 0, 200, 30, 0); 933 SetRateProfilePars(&rate_profile, 1, 400, 30, 150); 934 rate_profile.frame_index_rate_update[2] = kNbrFramesLong + 1; 935 rate_profile.num_frames = kNbrFramesLong; 936 // Codec/network settings. 937 CodecConfigPars process_settings; 938 SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, -1, 3, false, 939 true, true, false); 940 // Metrics for expected quality. 941 QualityMetrics quality_metrics; 942 SetQualityMetrics(&quality_metrics, 32.5, 30.0, 0.85, 0.80); 943 // Metrics for rate control. 944 RateControlMetrics rc_metrics[2]; 945 SetRateControlMetrics(rc_metrics, 0, 0, 20, 30, 10, 10, 0, 1); 946 SetRateControlMetrics(rc_metrics, 1, 0, 0, 30, 15, 10, 0, 0); 947 ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, 948 rc_metrics); 949 } 950 } // namespace webrtc 951
