1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include <assert.h> 12 #include <stdio.h> 13 #include <time.h> 14 15 #include <stdarg.h> 16 #include <sys/stat.h> // To check for directory existence. 17 18 #ifndef S_ISDIR // Not defined in stat.h on Windows. 19 #define S_ISDIR(mode) (((mode)&S_IFMT) == S_IFDIR) 20 #endif 21 22 #include "gflags/gflags.h" 23 #include "webrtc/base/format_macros.h" 24 #include "webrtc/common_types.h" 25 #include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h" 26 #include "webrtc/modules/video_coding/codecs/test/stats.h" 27 #include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" 28 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" 29 #include "webrtc/modules/video_coding/include/video_coding.h" 30 #include "webrtc/system_wrappers/include/trace.h" 31 #include "webrtc/test/testsupport/frame_reader.h" 32 #include "webrtc/test/testsupport/frame_writer.h" 33 #include "webrtc/test/testsupport/metrics/video_metrics.h" 34 #include "webrtc/test/testsupport/packet_reader.h" 35 36 DEFINE_string(test_name, "Quality test", "The name of the test to run. "); 37 DEFINE_string(test_description, 38 "", 39 "A more detailed description about what " 40 "the current test is about."); 41 DEFINE_string(input_filename, 42 "", 43 "Input file. " 44 "The source video file to be encoded and decoded. Must be in " 45 ".yuv format"); 46 DEFINE_int32(width, -1, "Width in pixels of the frames in the input file."); 47 DEFINE_int32(height, -1, "Height in pixels of the frames in the input file."); 48 DEFINE_int32(framerate, 49 30, 50 "Frame rate of the input file, in FPS " 51 "(frames-per-second). "); 52 DEFINE_string(output_dir, 53 ".", 54 "Output directory. " 55 "The directory where the output file will be put. Must already " 56 "exist."); 57 DEFINE_bool(use_single_core, 58 false, 59 "Force using a single core. If set to " 60 "true, only one core will be used for processing. Using a single " 61 "core is necessary to get a deterministic behavior for the" 62 "encoded frames - using multiple cores will produce different " 63 "encoded frames since multiple cores are competing to consume the " 64 "byte budget for each frame in parallel. If set to false, " 65 "the maximum detected number of cores will be used. "); 66 DEFINE_bool(disable_fixed_random_seed, 67 false, 68 "Set this flag to disable the" 69 "usage of a fixed random seed for the random generator used " 70 "for packet loss. Disabling this will cause consecutive runs " 71 "loose packets at different locations, which is bad for " 72 "reproducibility."); 73 DEFINE_string(output_filename, 74 "", 75 "Output file. " 76 "The name of the output video file resulting of the processing " 77 "of the source file. By default this is the same name as the " 78 "input file with '_out' appended before the extension."); 79 DEFINE_int32(bitrate, 500, "Bit rate in kilobits/second."); 80 DEFINE_int32(keyframe_interval, 81 0, 82 "Forces a keyframe every Nth frame. " 83 "0 means the encoder decides when to insert keyframes. Note that " 84 "the encoder may create a keyframe in other locations in addition " 85 "to the interval that is set using this parameter."); 86 DEFINE_int32(temporal_layers, 87 0, 88 "The number of temporal layers to use " 89 "(VP8 specific codec setting). Must be 0-4."); 90 DEFINE_int32(packet_size, 91 1500, 92 "Simulated network packet size in bytes (MTU). " 93 "Used for packet loss simulation."); 94 DEFINE_int32(max_payload_size, 95 1440, 96 "Max payload size in bytes for the " 97 "encoder."); 98 DEFINE_string(packet_loss_mode, 99 "uniform", 100 "Packet loss mode. Two different " 101 "packet loss models are supported: uniform or burst. This " 102 "setting has no effect unless packet_loss_rate is >0. "); 103 DEFINE_double(packet_loss_probability, 104 0.0, 105 "Packet loss probability. A value " 106 "between 0.0 and 1.0 that defines the probability of a packet " 107 "being lost. 0.1 means 10% and so on."); 108 DEFINE_int32(packet_loss_burst_length, 109 1, 110 "Packet loss burst length. Defines " 111 "how many packets will be lost in a burst when a packet has been " 112 "decided to be lost. Must be >=1."); 113 DEFINE_bool(csv, 114 false, 115 "CSV output. Enabling this will output all frame " 116 "statistics at the end of execution. Recommended to run combined " 117 "with --noverbose to avoid mixing output."); 118 DEFINE_bool(python, 119 false, 120 "Python output. Enabling this will output all frame " 121 "statistics as a Python script at the end of execution. " 122 "Recommended to run combine with --noverbose to avoid mixing " 123 "output."); 124 DEFINE_bool(verbose, 125 true, 126 "Verbose mode. Prints a lot of debugging info. " 127 "Suitable for tracking progress but not for capturing output. " 128 "Disable with --noverbose flag."); 129 130 // Custom log method that only prints if the verbose flag is given. 131 // Supports all the standard printf parameters and formatting (just forwarded). 132 int Log(const char* format, ...) { 133 int result = 0; 134 if (FLAGS_verbose) { 135 va_list args; 136 va_start(args, format); 137 result = vprintf(format, args); 138 va_end(args); 139 } 140 return result; 141 } 142 143 // Validates the arguments given as command line flags and fills in the 144 // TestConfig struct with all configurations needed for video processing. 145 // Returns 0 if everything is OK, otherwise an exit code. 146 int HandleCommandLineFlags(webrtc::test::TestConfig* config) { 147 // Validate the mandatory flags: 148 if (FLAGS_input_filename.empty() || FLAGS_width == -1 || FLAGS_height == -1) { 149 printf("%s\n", google::ProgramUsage()); 150 return 1; 151 } 152 config->name = FLAGS_test_name; 153 config->description = FLAGS_test_description; 154 155 // Verify the input file exists and is readable. 156 FILE* test_file; 157 test_file = fopen(FLAGS_input_filename.c_str(), "rb"); 158 if (test_file == NULL) { 159 fprintf(stderr, "Cannot read the specified input file: %s\n", 160 FLAGS_input_filename.c_str()); 161 return 2; 162 } 163 fclose(test_file); 164 config->input_filename = FLAGS_input_filename; 165 166 // Verify the output dir exists. 167 struct stat dir_info; 168 if (!(stat(FLAGS_output_dir.c_str(), &dir_info) == 0 && 169 S_ISDIR(dir_info.st_mode))) { 170 fprintf(stderr, "Cannot find output directory: %s\n", 171 FLAGS_output_dir.c_str()); 172 return 3; 173 } 174 config->output_dir = FLAGS_output_dir; 175 176 // Manufacture an output filename if none was given. 177 if (FLAGS_output_filename.empty()) { 178 // Cut out the filename without extension from the given input file 179 // (which may include a path) 180 int startIndex = FLAGS_input_filename.find_last_of("/") + 1; 181 if (startIndex == 0) { 182 startIndex = 0; 183 } 184 FLAGS_output_filename = 185 FLAGS_input_filename.substr( 186 startIndex, FLAGS_input_filename.find_last_of(".") - startIndex) + 187 "_out.yuv"; 188 } 189 190 // Verify output file can be written. 191 if (FLAGS_output_dir == ".") { 192 config->output_filename = FLAGS_output_filename; 193 } else { 194 config->output_filename = FLAGS_output_dir + "/" + FLAGS_output_filename; 195 } 196 test_file = fopen(config->output_filename.c_str(), "wb"); 197 if (test_file == NULL) { 198 fprintf(stderr, "Cannot write output file: %s\n", 199 config->output_filename.c_str()); 200 return 4; 201 } 202 fclose(test_file); 203 204 // Check single core flag. 205 config->use_single_core = FLAGS_use_single_core; 206 207 // Get codec specific configuration. 208 webrtc::VideoCodingModule::Codec(webrtc::kVideoCodecVP8, 209 config->codec_settings); 210 211 // Check the temporal layers. 212 if (FLAGS_temporal_layers < 0 || 213 FLAGS_temporal_layers > webrtc::kMaxTemporalStreams) { 214 fprintf(stderr, "Temporal layers number must be 0-4, was: %d\n", 215 FLAGS_temporal_layers); 216 return 13; 217 } 218 config->codec_settings->codecSpecific.VP8.numberOfTemporalLayers = 219 FLAGS_temporal_layers; 220 221 // Check the bit rate. 222 if (FLAGS_bitrate <= 0) { 223 fprintf(stderr, "Bit rate must be >0 kbps, was: %d\n", FLAGS_bitrate); 224 return 5; 225 } 226 config->codec_settings->startBitrate = FLAGS_bitrate; 227 228 // Check the keyframe interval. 229 if (FLAGS_keyframe_interval < 0) { 230 fprintf(stderr, "Keyframe interval must be >=0, was: %d\n", 231 FLAGS_keyframe_interval); 232 return 6; 233 } 234 config->keyframe_interval = FLAGS_keyframe_interval; 235 236 // Check packet size and max payload size. 237 if (FLAGS_packet_size <= 0) { 238 fprintf(stderr, "Packet size must be >0 bytes, was: %d\n", 239 FLAGS_packet_size); 240 return 7; 241 } 242 config->networking_config.packet_size_in_bytes = 243 static_cast<size_t>(FLAGS_packet_size); 244 245 if (FLAGS_max_payload_size <= 0) { 246 fprintf(stderr, "Max payload size must be >0 bytes, was: %d\n", 247 FLAGS_max_payload_size); 248 return 8; 249 } 250 config->networking_config.max_payload_size_in_bytes = 251 static_cast<size_t>(FLAGS_max_payload_size); 252 253 // Check the width and height 254 if (FLAGS_width <= 0 || FLAGS_height <= 0) { 255 fprintf(stderr, "Width and height must be >0."); 256 return 9; 257 } 258 config->codec_settings->width = FLAGS_width; 259 config->codec_settings->height = FLAGS_height; 260 config->codec_settings->maxFramerate = FLAGS_framerate; 261 262 // Calculate the size of each frame to read (according to YUV spec). 263 config->frame_length_in_bytes = 264 3 * config->codec_settings->width * config->codec_settings->height / 2; 265 266 // Check packet loss settings 267 if (FLAGS_packet_loss_mode != "uniform" && 268 FLAGS_packet_loss_mode != "burst") { 269 fprintf(stderr, 270 "Unsupported packet loss mode, must be 'uniform' or " 271 "'burst'\n."); 272 return 10; 273 } 274 config->networking_config.packet_loss_mode = webrtc::test::kUniform; 275 if (FLAGS_packet_loss_mode == "burst") { 276 config->networking_config.packet_loss_mode = webrtc::test::kBurst; 277 } 278 279 if (FLAGS_packet_loss_probability < 0.0 || 280 FLAGS_packet_loss_probability > 1.0) { 281 fprintf(stderr, 282 "Invalid packet loss probability. Must be 0.0 - 1.0, " 283 "was: %f\n", 284 FLAGS_packet_loss_probability); 285 return 11; 286 } 287 config->networking_config.packet_loss_probability = 288 FLAGS_packet_loss_probability; 289 290 if (FLAGS_packet_loss_burst_length < 1) { 291 fprintf(stderr, 292 "Invalid packet loss burst length, must be >=1, " 293 "was: %d\n", 294 FLAGS_packet_loss_burst_length); 295 return 12; 296 } 297 config->networking_config.packet_loss_burst_length = 298 FLAGS_packet_loss_burst_length; 299 config->verbose = FLAGS_verbose; 300 return 0; 301 } 302 303 void CalculateSsimVideoMetrics(webrtc::test::TestConfig* config, 304 webrtc::test::QualityMetricsResult* result) { 305 Log("Calculating SSIM...\n"); 306 I420SSIMFromFiles( 307 config->input_filename.c_str(), config->output_filename.c_str(), 308 config->codec_settings->width, config->codec_settings->height, result); 309 Log(" Average: %3.2f\n", result->average); 310 Log(" Min : %3.2f (frame %d)\n", result->min, result->min_frame_number); 311 Log(" Max : %3.2f (frame %d)\n", result->max, result->max_frame_number); 312 } 313 314 void CalculatePsnrVideoMetrics(webrtc::test::TestConfig* config, 315 webrtc::test::QualityMetricsResult* result) { 316 Log("Calculating PSNR...\n"); 317 I420PSNRFromFiles( 318 config->input_filename.c_str(), config->output_filename.c_str(), 319 config->codec_settings->width, config->codec_settings->height, result); 320 Log(" Average: %3.2f\n", result->average); 321 Log(" Min : %3.2f (frame %d)\n", result->min, result->min_frame_number); 322 Log(" Max : %3.2f (frame %d)\n", result->max, result->max_frame_number); 323 } 324 325 void PrintConfigurationSummary(const webrtc::test::TestConfig& config) { 326 Log("Quality test with parameters:\n"); 327 Log(" Test name : %s\n", config.name.c_str()); 328 Log(" Description : %s\n", config.description.c_str()); 329 Log(" Input filename : %s\n", config.input_filename.c_str()); 330 Log(" Output directory : %s\n", config.output_dir.c_str()); 331 Log(" Output filename : %s\n", config.output_filename.c_str()); 332 Log(" Frame length : %" PRIuS " bytes\n", config.frame_length_in_bytes); 333 Log(" Packet size : %" PRIuS " bytes\n", 334 config.networking_config.packet_size_in_bytes); 335 Log(" Max payload size : %" PRIuS " bytes\n", 336 config.networking_config.max_payload_size_in_bytes); 337 Log(" Packet loss:\n"); 338 Log(" Mode : %s\n", 339 PacketLossModeToStr(config.networking_config.packet_loss_mode)); 340 Log(" Probability : %2.1f\n", 341 config.networking_config.packet_loss_probability); 342 Log(" Burst length : %d packets\n", 343 config.networking_config.packet_loss_burst_length); 344 } 345 346 void PrintCsvOutput(const webrtc::test::Stats& stats, 347 const webrtc::test::QualityMetricsResult& ssim_result, 348 const webrtc::test::QualityMetricsResult& psnr_result) { 349 Log( 350 "\nCSV output (recommended to run with --noverbose to skip the " 351 "above output)\n"); 352 printf( 353 "frame_number encoding_successful decoding_successful " 354 "encode_return_code decode_return_code " 355 "encode_time_in_us decode_time_in_us " 356 "bit_rate_in_kbps encoded_frame_length_in_bytes frame_type " 357 "packets_dropped total_packets " 358 "ssim psnr\n"); 359 360 for (unsigned int i = 0; i < stats.stats_.size(); ++i) { 361 const webrtc::test::FrameStatistic& f = stats.stats_[i]; 362 const webrtc::test::FrameResult& ssim = ssim_result.frames[i]; 363 const webrtc::test::FrameResult& psnr = psnr_result.frames[i]; 364 printf("%4d, %d, %d, %2d, %2d, %6d, %6d, %5d, %7" PRIuS 365 ", %d, %2d, %2" PRIuS ", %5.3f, %5.2f\n", 366 f.frame_number, f.encoding_successful, f.decoding_successful, 367 f.encode_return_code, f.decode_return_code, f.encode_time_in_us, 368 f.decode_time_in_us, f.bit_rate_in_kbps, 369 f.encoded_frame_length_in_bytes, f.frame_type, f.packets_dropped, 370 f.total_packets, ssim.value, psnr.value); 371 } 372 } 373 374 void PrintPythonOutput(const webrtc::test::TestConfig& config, 375 const webrtc::test::Stats& stats, 376 const webrtc::test::QualityMetricsResult& ssim_result, 377 const webrtc::test::QualityMetricsResult& psnr_result) { 378 Log( 379 "\nPython output (recommended to run with --noverbose to skip the " 380 "above output)\n"); 381 printf( 382 "test_configuration = [" 383 "{'name': 'name', 'value': '%s'},\n" 384 "{'name': 'description', 'value': '%s'},\n" 385 "{'name': 'test_number', 'value': '%d'},\n" 386 "{'name': 'input_filename', 'value': '%s'},\n" 387 "{'name': 'output_filename', 'value': '%s'},\n" 388 "{'name': 'output_dir', 'value': '%s'},\n" 389 "{'name': 'packet_size_in_bytes', 'value': '%" PRIuS 390 "'},\n" 391 "{'name': 'max_payload_size_in_bytes', 'value': '%" PRIuS 392 "'},\n" 393 "{'name': 'packet_loss_mode', 'value': '%s'},\n" 394 "{'name': 'packet_loss_probability', 'value': '%f'},\n" 395 "{'name': 'packet_loss_burst_length', 'value': '%d'},\n" 396 "{'name': 'exclude_frame_types', 'value': '%s'},\n" 397 "{'name': 'frame_length_in_bytes', 'value': '%" PRIuS 398 "'},\n" 399 "{'name': 'use_single_core', 'value': '%s'},\n" 400 "{'name': 'keyframe_interval;', 'value': '%d'},\n" 401 "{'name': 'video_codec_type', 'value': '%s'},\n" 402 "{'name': 'width', 'value': '%d'},\n" 403 "{'name': 'height', 'value': '%d'},\n" 404 "{'name': 'bit_rate_in_kbps', 'value': '%d'},\n" 405 "]\n", 406 config.name.c_str(), config.description.c_str(), config.test_number, 407 config.input_filename.c_str(), config.output_filename.c_str(), 408 config.output_dir.c_str(), config.networking_config.packet_size_in_bytes, 409 config.networking_config.max_payload_size_in_bytes, 410 PacketLossModeToStr(config.networking_config.packet_loss_mode), 411 config.networking_config.packet_loss_probability, 412 config.networking_config.packet_loss_burst_length, 413 ExcludeFrameTypesToStr(config.exclude_frame_types), 414 config.frame_length_in_bytes, config.use_single_core ? "True " : "False", 415 config.keyframe_interval, 416 webrtc::test::VideoCodecTypeToStr(config.codec_settings->codecType), 417 config.codec_settings->width, config.codec_settings->height, 418 config.codec_settings->startBitrate); 419 printf( 420 "frame_data_types = {" 421 "'frame_number': ('number', 'Frame number'),\n" 422 "'encoding_successful': ('boolean', 'Encoding successful?'),\n" 423 "'decoding_successful': ('boolean', 'Decoding successful?'),\n" 424 "'encode_time': ('number', 'Encode time (us)'),\n" 425 "'decode_time': ('number', 'Decode time (us)'),\n" 426 "'encode_return_code': ('number', 'Encode return code'),\n" 427 "'decode_return_code': ('number', 'Decode return code'),\n" 428 "'bit_rate': ('number', 'Bit rate (kbps)'),\n" 429 "'encoded_frame_length': " 430 "('number', 'Encoded frame length (bytes)'),\n" 431 "'frame_type': ('string', 'Frame type'),\n" 432 "'packets_dropped': ('number', 'Packets dropped'),\n" 433 "'total_packets': ('number', 'Total packets'),\n" 434 "'ssim': ('number', 'SSIM'),\n" 435 "'psnr': ('number', 'PSNR (dB)'),\n" 436 "}\n"); 437 printf("frame_data = ["); 438 for (unsigned int i = 0; i < stats.stats_.size(); ++i) { 439 const webrtc::test::FrameStatistic& f = stats.stats_[i]; 440 const webrtc::test::FrameResult& ssim = ssim_result.frames[i]; 441 const webrtc::test::FrameResult& psnr = psnr_result.frames[i]; 442 printf( 443 "{'frame_number': %d, " 444 "'encoding_successful': %s, 'decoding_successful': %s, " 445 "'encode_time': %d, 'decode_time': %d, " 446 "'encode_return_code': %d, 'decode_return_code': %d, " 447 "'bit_rate': %d, 'encoded_frame_length': %" PRIuS 448 ", " 449 "'frame_type': %s, 'packets_dropped': %d, " 450 "'total_packets': %" PRIuS ", 'ssim': %f, 'psnr': %f},\n", 451 f.frame_number, f.encoding_successful ? "True " : "False", 452 f.decoding_successful ? "True " : "False", f.encode_time_in_us, 453 f.decode_time_in_us, f.encode_return_code, f.decode_return_code, 454 f.bit_rate_in_kbps, f.encoded_frame_length_in_bytes, 455 f.frame_type == webrtc::kVideoFrameDelta ? "'Delta'" : "'Other'", 456 f.packets_dropped, f.total_packets, ssim.value, psnr.value); 457 } 458 printf("]\n"); 459 } 460 461 // Runs a quality measurement on the input file supplied to the program. 462 // The input file must be in YUV format. 463 int main(int argc, char* argv[]) { 464 std::string program_name = argv[0]; 465 std::string usage = 466 "Quality test application for video comparisons.\n" 467 "Run " + 468 program_name + 469 " --helpshort for usage.\n" 470 "Example usage:\n" + 471 program_name + 472 " --input_filename=filename.yuv --width=352 --height=288\n"; 473 google::SetUsageMessage(usage); 474 475 google::ParseCommandLineFlags(&argc, &argv, true); 476 477 // Create TestConfig and codec settings struct. 478 webrtc::test::TestConfig config; 479 webrtc::VideoCodec codec_settings; 480 config.codec_settings = &codec_settings; 481 482 int return_code = HandleCommandLineFlags(&config); 483 // Exit if an invalid argument is supplied. 484 if (return_code != 0) { 485 return return_code; 486 } 487 488 PrintConfigurationSummary(config); 489 490 webrtc::VP8Encoder* encoder = webrtc::VP8Encoder::Create(); 491 webrtc::VP8Decoder* decoder = webrtc::VP8Decoder::Create(); 492 webrtc::test::Stats stats; 493 webrtc::test::FrameReaderImpl frame_reader(config.input_filename, 494 config.frame_length_in_bytes); 495 webrtc::test::FrameWriterImpl frame_writer(config.output_filename, 496 config.frame_length_in_bytes); 497 frame_reader.Init(); 498 frame_writer.Init(); 499 webrtc::test::PacketReader packet_reader; 500 501 webrtc::test::PacketManipulatorImpl packet_manipulator( 502 &packet_reader, config.networking_config, config.verbose); 503 // By default the packet manipulator is seeded with a fixed random. 504 // If disabled we must generate a new seed. 505 if (FLAGS_disable_fixed_random_seed) { 506 packet_manipulator.InitializeRandomSeed(time(NULL)); 507 } 508 webrtc::test::VideoProcessor* processor = 509 new webrtc::test::VideoProcessorImpl(encoder, decoder, &frame_reader, 510 &frame_writer, &packet_manipulator, 511 config, &stats); 512 processor->Init(); 513 514 int frame_number = 0; 515 while (processor->ProcessFrame(frame_number)) { 516 if (frame_number % 80 == 0) { 517 Log("\n"); // make the output a bit nicer. 518 } 519 Log("."); 520 frame_number++; 521 } 522 Log("\n"); 523 Log("Processed %d frames\n", frame_number); 524 525 // Release encoder and decoder to make sure they have finished processing. 526 encoder->Release(); 527 decoder->Release(); 528 529 // Verify statistics are correct: 530 assert(frame_number == static_cast<int>(stats.stats_.size())); 531 532 // Close the files before we start using them for SSIM/PSNR calculations. 533 frame_reader.Close(); 534 frame_writer.Close(); 535 536 stats.PrintSummary(); 537 538 webrtc::test::QualityMetricsResult ssim_result; 539 CalculateSsimVideoMetrics(&config, &ssim_result); 540 webrtc::test::QualityMetricsResult psnr_result; 541 CalculatePsnrVideoMetrics(&config, &psnr_result); 542 543 if (FLAGS_csv) { 544 PrintCsvOutput(stats, ssim_result, psnr_result); 545 } 546 if (FLAGS_python) { 547 PrintPythonOutput(config, stats, ssim_result, psnr_result); 548 } 549 delete processor; 550 delete encoder; 551 delete decoder; 552 Log("Quality test finished!"); 553 return 0; 554 } 555
