1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include <assert.h> 12 #include <stdio.h> 13 #include <time.h> 14 15 #include <stdarg.h> 16 #include <sys/stat.h> // To check for directory existence. 17 18 #ifndef S_ISDIR // Not defined in stat.h on Windows. 19 #define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) 20 #endif 21 22 #include "gflags/gflags.h" 23 #include "webrtc/common_types.h" 24 #include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h" 25 #include "webrtc/modules/video_coding/codecs/test/stats.h" 26 #include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" 27 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" 28 #include "webrtc/modules/video_coding/main/interface/video_coding.h" 29 #include "webrtc/system_wrappers/interface/trace.h" 30 #include "webrtc/test/testsupport/frame_reader.h" 31 #include "webrtc/test/testsupport/frame_writer.h" 32 #include "webrtc/test/testsupport/metrics/video_metrics.h" 33 #include "webrtc/test/testsupport/packet_reader.h" 34 35 DEFINE_string(test_name, "Quality test", "The name of the test to run. "); 36 DEFINE_string(test_description, "", "A more detailed description about what " 37 "the current test is about."); 38 DEFINE_string(input_filename, "", "Input file. " 39 "The source video file to be encoded and decoded. Must be in " 40 ".yuv format"); 41 DEFINE_int32(width, -1, "Width in pixels of the frames in the input file."); 42 DEFINE_int32(height, -1, "Height in pixels of the frames in the input file."); 43 DEFINE_int32(framerate, 30, "Frame rate of the input file, in FPS " 44 "(frames-per-second). "); 45 DEFINE_string(output_dir, ".", "Output directory. " 46 "The directory where the output file will be put. Must already " 47 "exist."); 48 DEFINE_bool(use_single_core, false, "Force using a single core. If set to " 49 "true, only one core will be used for processing. Using a single " 50 "core is necessary to get a deterministic behavior for the" 51 "encoded frames - using multiple cores will produce different " 52 "encoded frames since multiple cores are competing to consume the " 53 "byte budget for each frame in parallel. If set to false, " 54 "the maximum detected number of cores will be used. "); 55 DEFINE_bool(disable_fixed_random_seed , false, "Set this flag to disable the" 56 "usage of a fixed random seed for the random generator used " 57 "for packet loss. Disabling this will cause consecutive runs " 58 "loose packets at different locations, which is bad for " 59 "reproducibility."); 60 DEFINE_string(output_filename, "", "Output file. " 61 "The name of the output video file resulting of the processing " 62 "of the source file. By default this is the same name as the " 63 "input file with '_out' appended before the extension."); 64 DEFINE_int32(bitrate, 500, "Bit rate in kilobits/second."); 65 DEFINE_int32(keyframe_interval, 0, "Forces a keyframe every Nth frame. " 66 "0 means the encoder decides when to insert keyframes. Note that " 67 "the encoder may create a keyframe in other locations in addition " 68 "to the interval that is set using this parameter."); 69 DEFINE_int32(temporal_layers, 0, "The number of temporal layers to use " 70 "(VP8 specific codec setting). Must be 0-4."); 71 DEFINE_int32(packet_size, 1500, "Simulated network packet size in bytes (MTU). " 72 "Used for packet loss simulation."); 73 DEFINE_int32(max_payload_size, 1440, "Max payload size in bytes for the " 74 "encoder."); 75 DEFINE_string(packet_loss_mode, "uniform", "Packet loss mode. Two different " 76 "packet loss models are supported: uniform or burst. This " 77 "setting has no effect unless packet_loss_rate is >0. "); 78 DEFINE_double(packet_loss_probability, 0.0, "Packet loss probability. A value " 79 "between 0.0 and 1.0 that defines the probability of a packet " 80 "being lost. 0.1 means 10% and so on."); 81 DEFINE_int32(packet_loss_burst_length, 1, "Packet loss burst length. Defines " 82 "how many packets will be lost in a burst when a packet has been " 83 "decided to be lost. Must be >=1."); 84 DEFINE_bool(csv, false, "CSV output. Enabling this will output all frame " 85 "statistics at the end of execution. Recommended to run combined " 86 "with --noverbose to avoid mixing output."); 87 DEFINE_bool(python, false, "Python output. Enabling this will output all frame " 88 "statistics as a Python script at the end of execution. " 89 "Recommended to run combine with --noverbose to avoid mixing " 90 "output."); 91 DEFINE_bool(verbose, true, "Verbose mode. Prints a lot of debugging info. " 92 "Suitable for tracking progress but not for capturing output. " 93 "Disable with --noverbose flag."); 94 95 // Custom log method that only prints if the verbose flag is given. 96 // Supports all the standard printf parameters and formatting (just forwarded). 97 int Log(const char *format, ...) { 98 int result = 0; 99 if (FLAGS_verbose) { 100 va_list args; 101 va_start(args, format); 102 result = vprintf(format, args); 103 va_end(args); 104 } 105 return result; 106 } 107 108 // Validates the arguments given as command line flags and fills in the 109 // TestConfig struct with all configurations needed for video processing. 110 // Returns 0 if everything is OK, otherwise an exit code. 111 int HandleCommandLineFlags(webrtc::test::TestConfig* config) { 112 // Validate the mandatory flags: 113 if (FLAGS_input_filename == "" || FLAGS_width == -1 || FLAGS_height == -1) { 114 printf("%s\n", google::ProgramUsage()); 115 return 1; 116 } 117 config->name = FLAGS_test_name; 118 config->description = FLAGS_test_description; 119 120 // Verify the input file exists and is readable. 121 FILE* test_file; 122 test_file = fopen(FLAGS_input_filename.c_str(), "rb"); 123 if (test_file == NULL) { 124 fprintf(stderr, "Cannot read the specified input file: %s\n", 125 FLAGS_input_filename.c_str()); 126 return 2; 127 } 128 fclose(test_file); 129 config->input_filename = FLAGS_input_filename; 130 131 // Verify the output dir exists. 132 struct stat dir_info; 133 if (!(stat(FLAGS_output_dir.c_str(), &dir_info) == 0 && 134 S_ISDIR(dir_info.st_mode))) { 135 fprintf(stderr, "Cannot find output directory: %s\n", 136 FLAGS_output_dir.c_str()); 137 return 3; 138 } 139 config->output_dir = FLAGS_output_dir; 140 141 // Manufacture an output filename if none was given. 142 if (FLAGS_output_filename == "") { 143 // Cut out the filename without extension from the given input file 144 // (which may include a path) 145 int startIndex = FLAGS_input_filename.find_last_of("/") + 1; 146 if (startIndex == 0) { 147 startIndex = 0; 148 } 149 FLAGS_output_filename = 150 FLAGS_input_filename.substr(startIndex, 151 FLAGS_input_filename.find_last_of(".") 152 - startIndex) + "_out.yuv"; 153 } 154 155 // Verify output file can be written. 156 if (FLAGS_output_dir == ".") { 157 config->output_filename = FLAGS_output_filename; 158 } else { 159 config->output_filename = FLAGS_output_dir + "/"+ FLAGS_output_filename; 160 } 161 test_file = fopen(config->output_filename.c_str(), "wb"); 162 if (test_file == NULL) { 163 fprintf(stderr, "Cannot write output file: %s\n", 164 config->output_filename.c_str()); 165 return 4; 166 } 167 fclose(test_file); 168 169 // Check single core flag. 170 config->use_single_core = FLAGS_use_single_core; 171 172 // Get codec specific configuration. 173 webrtc::VideoCodingModule::Codec(webrtc::kVideoCodecVP8, 174 config->codec_settings); 175 176 // Check the temporal layers. 177 if (FLAGS_temporal_layers < 0 || 178 FLAGS_temporal_layers > webrtc::kMaxTemporalStreams) { 179 fprintf(stderr, "Temporal layers number must be 0-4, was: %d\n", 180 FLAGS_temporal_layers); 181 return 13; 182 } 183 config->codec_settings->codecSpecific.VP8.numberOfTemporalLayers = 184 FLAGS_temporal_layers; 185 186 // Check the bit rate. 187 if (FLAGS_bitrate <= 0) { 188 fprintf(stderr, "Bit rate must be >0 kbps, was: %d\n", FLAGS_bitrate); 189 return 5; 190 } 191 config->codec_settings->startBitrate = FLAGS_bitrate; 192 193 // Check the keyframe interval. 194 if (FLAGS_keyframe_interval < 0) { 195 fprintf(stderr, "Keyframe interval must be >=0, was: %d\n", 196 FLAGS_keyframe_interval); 197 return 6; 198 } 199 config->keyframe_interval = FLAGS_keyframe_interval; 200 201 // Check packet size and max payload size. 202 if (FLAGS_packet_size <= 0) { 203 fprintf(stderr, "Packet size must be >0 bytes, was: %d\n", 204 FLAGS_packet_size); 205 return 7; 206 } 207 config->networking_config.packet_size_in_bytes = FLAGS_packet_size; 208 209 if (FLAGS_max_payload_size <= 0) { 210 fprintf(stderr, "Max payload size must be >0 bytes, was: %d\n", 211 FLAGS_max_payload_size); 212 return 8; 213 } 214 config->networking_config.max_payload_size_in_bytes = 215 FLAGS_max_payload_size; 216 217 // Check the width and height 218 if (FLAGS_width <= 0 || FLAGS_height <= 0) { 219 fprintf(stderr, "Width and height must be >0."); 220 return 9; 221 } 222 config->codec_settings->width = FLAGS_width; 223 config->codec_settings->height = FLAGS_height; 224 config->codec_settings->maxFramerate = FLAGS_framerate; 225 226 // Calculate the size of each frame to read (according to YUV spec). 227 config->frame_length_in_bytes = 228 3 * config->codec_settings->width * config->codec_settings->height / 2; 229 230 // Check packet loss settings 231 if (FLAGS_packet_loss_mode != "uniform" && 232 FLAGS_packet_loss_mode != "burst") { 233 fprintf(stderr, "Unsupported packet loss mode, must be 'uniform' or " 234 "'burst'\n."); 235 return 10; 236 } 237 config->networking_config.packet_loss_mode = webrtc::test::kUniform; 238 if (FLAGS_packet_loss_mode == "burst") { 239 config->networking_config.packet_loss_mode = webrtc::test::kBurst; 240 } 241 242 if (FLAGS_packet_loss_probability < 0.0 || 243 FLAGS_packet_loss_probability > 1.0) { 244 fprintf(stderr, "Invalid packet loss probability. Must be 0.0 - 1.0, " 245 "was: %f\n", FLAGS_packet_loss_probability); 246 return 11; 247 } 248 config->networking_config.packet_loss_probability = 249 FLAGS_packet_loss_probability; 250 251 if (FLAGS_packet_loss_burst_length < 1) { 252 fprintf(stderr, "Invalid packet loss burst length, must be >=1, " 253 "was: %d\n", FLAGS_packet_loss_burst_length); 254 return 12; 255 } 256 config->networking_config.packet_loss_burst_length = 257 FLAGS_packet_loss_burst_length; 258 config->verbose = FLAGS_verbose; 259 return 0; 260 } 261 262 void CalculateSsimVideoMetrics(webrtc::test::TestConfig* config, 263 webrtc::test::QualityMetricsResult* result) { 264 Log("Calculating SSIM...\n"); 265 I420SSIMFromFiles(config->input_filename.c_str(), 266 config->output_filename.c_str(), 267 config->codec_settings->width, 268 config->codec_settings->height, result); 269 Log(" Average: %3.2f\n", result->average); 270 Log(" Min : %3.2f (frame %d)\n", result->min, result->min_frame_number); 271 Log(" Max : %3.2f (frame %d)\n", result->max, result->max_frame_number); 272 } 273 274 void CalculatePsnrVideoMetrics(webrtc::test::TestConfig* config, 275 webrtc::test::QualityMetricsResult* result) { 276 Log("Calculating PSNR...\n"); 277 I420PSNRFromFiles(config->input_filename.c_str(), 278 config->output_filename.c_str(), 279 config->codec_settings->width, 280 config->codec_settings->height, result); 281 Log(" Average: %3.2f\n", result->average); 282 Log(" Min : %3.2f (frame %d)\n", result->min, result->min_frame_number); 283 Log(" Max : %3.2f (frame %d)\n", result->max, result->max_frame_number); 284 } 285 286 void PrintConfigurationSummary(const webrtc::test::TestConfig& config) { 287 Log("Quality test with parameters:\n"); 288 Log(" Test name : %s\n", config.name.c_str()); 289 Log(" Description : %s\n", config.description.c_str()); 290 Log(" Input filename : %s\n", config.input_filename.c_str()); 291 Log(" Output directory : %s\n", config.output_dir.c_str()); 292 Log(" Output filename : %s\n", config.output_filename.c_str()); 293 Log(" Frame length : %d bytes\n", config.frame_length_in_bytes); 294 Log(" Packet size : %d bytes\n", 295 config.networking_config.packet_size_in_bytes); 296 Log(" Max payload size : %d bytes\n", 297 config.networking_config.max_payload_size_in_bytes); 298 Log(" Packet loss:\n"); 299 Log(" Mode : %s\n", 300 PacketLossModeToStr(config.networking_config.packet_loss_mode)); 301 Log(" Probability : %2.1f\n", 302 config.networking_config.packet_loss_probability); 303 Log(" Burst length : %d packets\n", 304 config.networking_config.packet_loss_burst_length); 305 } 306 307 void PrintCsvOutput(const webrtc::test::Stats& stats, 308 const webrtc::test::QualityMetricsResult& ssim_result, 309 const webrtc::test::QualityMetricsResult& psnr_result) { 310 Log("\nCSV output (recommended to run with --noverbose to skip the " 311 "above output)\n"); 312 printf("frame_number encoding_successful decoding_successful " 313 "encode_return_code decode_return_code " 314 "encode_time_in_us decode_time_in_us " 315 "bit_rate_in_kbps encoded_frame_length_in_bytes frame_type " 316 "packets_dropped total_packets " 317 "ssim psnr\n"); 318 319 for (unsigned int i = 0; i < stats.stats_.size(); ++i) { 320 const webrtc::test::FrameStatistic& f = stats.stats_[i]; 321 const webrtc::test::FrameResult& ssim = ssim_result.frames[i]; 322 const webrtc::test::FrameResult& psnr = psnr_result.frames[i]; 323 printf("%4d, %d, %d, %2d, %2d, %6d, %6d, %5d, %7d, %d, %2d, %2d, " 324 "%5.3f, %5.2f\n", 325 f.frame_number, 326 f.encoding_successful, 327 f.decoding_successful, 328 f.encode_return_code, 329 f.decode_return_code, 330 f.encode_time_in_us, 331 f.decode_time_in_us, 332 f.bit_rate_in_kbps, 333 f.encoded_frame_length_in_bytes, 334 f.frame_type, 335 f.packets_dropped, 336 f.total_packets, 337 ssim.value, 338 psnr.value); 339 } 340 } 341 342 void PrintPythonOutput(const webrtc::test::TestConfig& config, 343 const webrtc::test::Stats& stats, 344 const webrtc::test::QualityMetricsResult& ssim_result, 345 const webrtc::test::QualityMetricsResult& psnr_result) { 346 Log("\nPython output (recommended to run with --noverbose to skip the " 347 "above output)\n"); 348 printf("test_configuration = [" 349 "{'name': 'name', 'value': '%s'},\n" 350 "{'name': 'description', 'value': '%s'},\n" 351 "{'name': 'test_number', 'value': '%d'},\n" 352 "{'name': 'input_filename', 'value': '%s'},\n" 353 "{'name': 'output_filename', 'value': '%s'},\n" 354 "{'name': 'output_dir', 'value': '%s'},\n" 355 "{'name': 'packet_size_in_bytes', 'value': '%d'},\n" 356 "{'name': 'max_payload_size_in_bytes', 'value': '%d'},\n" 357 "{'name': 'packet_loss_mode', 'value': '%s'},\n" 358 "{'name': 'packet_loss_probability', 'value': '%f'},\n" 359 "{'name': 'packet_loss_burst_length', 'value': '%d'},\n" 360 "{'name': 'exclude_frame_types', 'value': '%s'},\n" 361 "{'name': 'frame_length_in_bytes', 'value': '%d'},\n" 362 "{'name': 'use_single_core', 'value': '%s'},\n" 363 "{'name': 'keyframe_interval;', 'value': '%d'},\n" 364 "{'name': 'video_codec_type', 'value': '%s'},\n" 365 "{'name': 'width', 'value': '%d'},\n" 366 "{'name': 'height', 'value': '%d'},\n" 367 "{'name': 'bit_rate_in_kbps', 'value': '%d'},\n" 368 "]\n", 369 config.name.c_str(), 370 config.description.c_str(), 371 config.test_number, 372 config.input_filename.c_str(), 373 config.output_filename.c_str(), 374 config.output_dir.c_str(), 375 config.networking_config.packet_size_in_bytes, 376 config.networking_config.max_payload_size_in_bytes, 377 PacketLossModeToStr(config.networking_config.packet_loss_mode), 378 config.networking_config.packet_loss_probability, 379 config.networking_config.packet_loss_burst_length, 380 ExcludeFrameTypesToStr(config.exclude_frame_types), 381 config.frame_length_in_bytes, 382 config.use_single_core ? "True " : "False", 383 config.keyframe_interval, 384 webrtc::test::VideoCodecTypeToStr(config.codec_settings->codecType), 385 config.codec_settings->width, 386 config.codec_settings->height, 387 config.codec_settings->startBitrate); 388 printf("frame_data_types = {" 389 "'frame_number': ('number', 'Frame number'),\n" 390 "'encoding_successful': ('boolean', 'Encoding successful?'),\n" 391 "'decoding_successful': ('boolean', 'Decoding successful?'),\n" 392 "'encode_time': ('number', 'Encode time (us)'),\n" 393 "'decode_time': ('number', 'Decode time (us)'),\n" 394 "'encode_return_code': ('number', 'Encode return code'),\n" 395 "'decode_return_code': ('number', 'Decode return code'),\n" 396 "'bit_rate': ('number', 'Bit rate (kbps)'),\n" 397 "'encoded_frame_length': " 398 "('number', 'Encoded frame length (bytes)'),\n" 399 "'frame_type': ('string', 'Frame type'),\n" 400 "'packets_dropped': ('number', 'Packets dropped'),\n" 401 "'total_packets': ('number', 'Total packets'),\n" 402 "'ssim': ('number', 'SSIM'),\n" 403 "'psnr': ('number', 'PSNR (dB)'),\n" 404 "}\n"); 405 printf("frame_data = ["); 406 for (unsigned int i = 0; i < stats.stats_.size(); ++i) { 407 const webrtc::test::FrameStatistic& f = stats.stats_[i]; 408 const webrtc::test::FrameResult& ssim = ssim_result.frames[i]; 409 const webrtc::test::FrameResult& psnr = psnr_result.frames[i]; 410 printf("{'frame_number': %d, " 411 "'encoding_successful': %s, 'decoding_successful': %s, " 412 "'encode_time': %d, 'decode_time': %d, " 413 "'encode_return_code': %d, 'decode_return_code': %d, " 414 "'bit_rate': %d, 'encoded_frame_length': %d, 'frame_type': %s, " 415 "'packets_dropped': %d, 'total_packets': %d, " 416 "'ssim': %f, 'psnr': %f},\n", 417 f.frame_number, 418 f.encoding_successful ? "True " : "False", 419 f.decoding_successful ? "True " : "False", 420 f.encode_time_in_us, 421 f.decode_time_in_us, 422 f.encode_return_code, 423 f.decode_return_code, 424 f.bit_rate_in_kbps, 425 f.encoded_frame_length_in_bytes, 426 f.frame_type == webrtc::kDeltaFrame ? "'Delta'" : "'Other'", 427 f.packets_dropped, 428 f.total_packets, 429 ssim.value, 430 psnr.value); 431 } 432 printf("]\n"); 433 } 434 435 // Runs a quality measurement on the input file supplied to the program. 436 // The input file must be in YUV format. 437 int main(int argc, char* argv[]) { 438 std::string program_name = argv[0]; 439 std::string usage = "Quality test application for video comparisons.\n" 440 "Run " + program_name + " --helpshort for usage.\n" 441 "Example usage:\n" + program_name + 442 " --input_filename=filename.yuv --width=352 --height=288\n"; 443 google::SetUsageMessage(usage); 444 445 google::ParseCommandLineFlags(&argc, &argv, true); 446 447 // Create TestConfig and codec settings struct. 448 webrtc::test::TestConfig config; 449 webrtc::VideoCodec codec_settings; 450 config.codec_settings = &codec_settings; 451 452 int return_code = HandleCommandLineFlags(&config); 453 // Exit if an invalid argument is supplied. 454 if (return_code != 0) { 455 return return_code; 456 } 457 458 PrintConfigurationSummary(config); 459 460 webrtc::VP8Encoder* encoder = webrtc::VP8Encoder::Create(); 461 webrtc::VP8Decoder* decoder = webrtc::VP8Decoder::Create(); 462 webrtc::test::Stats stats; 463 webrtc::test::FrameReaderImpl frame_reader(config.input_filename, 464 config.frame_length_in_bytes); 465 webrtc::test::FrameWriterImpl frame_writer(config.output_filename, 466 config.frame_length_in_bytes); 467 frame_reader.Init(); 468 frame_writer.Init(); 469 webrtc::test::PacketReader packet_reader; 470 471 webrtc::test::PacketManipulatorImpl packet_manipulator( 472 &packet_reader, config.networking_config, config.verbose); 473 // By default the packet manipulator is seeded with a fixed random. 474 // If disabled we must generate a new seed. 475 if (FLAGS_disable_fixed_random_seed) { 476 packet_manipulator.InitializeRandomSeed(time(NULL)); 477 } 478 webrtc::test::VideoProcessor* processor = 479 new webrtc::test::VideoProcessorImpl(encoder, decoder, 480 &frame_reader, 481 &frame_writer, 482 &packet_manipulator, 483 config, &stats); 484 processor->Init(); 485 486 int frame_number = 0; 487 while (processor->ProcessFrame(frame_number)) { 488 if (frame_number % 80 == 0) { 489 Log("\n"); // make the output a bit nicer. 490 } 491 Log("."); 492 frame_number++; 493 } 494 Log("\n"); 495 Log("Processed %d frames\n", frame_number); 496 497 // Release encoder and decoder to make sure they have finished processing. 498 encoder->Release(); 499 decoder->Release(); 500 501 // Verify statistics are correct: 502 assert(frame_number == static_cast<int>(stats.stats_.size())); 503 504 // Close the files before we start using them for SSIM/PSNR calculations. 505 frame_reader.Close(); 506 frame_writer.Close(); 507 508 stats.PrintSummary(); 509 510 webrtc::test::QualityMetricsResult ssim_result; 511 CalculateSsimVideoMetrics(&config, &ssim_result); 512 webrtc::test::QualityMetricsResult psnr_result; 513 CalculatePsnrVideoMetrics(&config, &psnr_result); 514 515 if (FLAGS_csv) { 516 PrintCsvOutput(stats, ssim_result, psnr_result); 517 } 518 if (FLAGS_python) { 519 PrintPythonOutput(config, stats, ssim_result, psnr_result); 520 } 521 delete processor; 522 delete encoder; 523 delete decoder; 524 Log("Quality test finished!"); 525 return 0; 526 } 527
