1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" 12 13 #include <assert.h> 14 #include <string.h> 15 16 #include <limits> 17 #include <vector> 18 19 #include "webrtc/system_wrappers/include/cpu_info.h" 20 21 namespace webrtc { 22 namespace test { 23 24 TestConfig::TestConfig() 25 : name(""), 26 description(""), 27 test_number(0), 28 input_filename(""), 29 output_filename(""), 30 output_dir("out"), 31 networking_config(), 32 exclude_frame_types(kExcludeOnlyFirstKeyFrame), 33 frame_length_in_bytes(0), 34 use_single_core(false), 35 keyframe_interval(0), 36 codec_settings(NULL), 37 verbose(true) {} 38 39 TestConfig::~TestConfig() {} 40 41 VideoProcessorImpl::VideoProcessorImpl(webrtc::VideoEncoder* encoder, 42 webrtc::VideoDecoder* decoder, 43 FrameReader* frame_reader, 44 FrameWriter* frame_writer, 45 PacketManipulator* packet_manipulator, 46 const TestConfig& config, 47 Stats* stats) 48 : encoder_(encoder), 49 decoder_(decoder), 50 frame_reader_(frame_reader), 51 frame_writer_(frame_writer), 52 packet_manipulator_(packet_manipulator), 53 config_(config), 54 stats_(stats), 55 encode_callback_(NULL), 56 decode_callback_(NULL), 57 source_buffer_(NULL), 58 first_key_frame_has_been_excluded_(false), 59 last_frame_missing_(false), 60 initialized_(false), 61 encoded_frame_size_(0), 62 encoded_frame_type_(kVideoFrameKey), 63 prev_time_stamp_(0), 64 num_dropped_frames_(0), 65 num_spatial_resizes_(0), 66 last_encoder_frame_width_(0), 67 last_encoder_frame_height_(0), 68 scaler_() { 69 assert(encoder); 70 assert(decoder); 71 assert(frame_reader); 72 assert(frame_writer); 73 assert(packet_manipulator); 74 assert(stats); 75 } 76 77 bool VideoProcessorImpl::Init() { 78 // Calculate a factor used for bit rate calculations: 79 bit_rate_factor_ = config_.codec_settings->maxFramerate * 0.001 * 8; // bits 80 81 // Initialize data structures used by the encoder/decoder APIs 82 size_t frame_length_in_bytes = frame_reader_->FrameLength(); 83 source_buffer_ = new uint8_t[frame_length_in_bytes]; 84 last_successful_frame_buffer_ = new uint8_t[frame_length_in_bytes]; 85 // Set fixed properties common for all frames. 86 // To keep track of spatial resize actions by encoder. 87 last_encoder_frame_width_ = config_.codec_settings->width; 88 last_encoder_frame_height_ = config_.codec_settings->height; 89 90 // Setup required callbacks for the encoder/decoder: 91 encode_callback_ = new VideoProcessorEncodeCompleteCallback(this); 92 decode_callback_ = new VideoProcessorDecodeCompleteCallback(this); 93 int32_t register_result = 94 encoder_->RegisterEncodeCompleteCallback(encode_callback_); 95 if (register_result != WEBRTC_VIDEO_CODEC_OK) { 96 fprintf(stderr, 97 "Failed to register encode complete callback, return code: " 98 "%d\n", 99 register_result); 100 return false; 101 } 102 register_result = decoder_->RegisterDecodeCompleteCallback(decode_callback_); 103 if (register_result != WEBRTC_VIDEO_CODEC_OK) { 104 fprintf(stderr, 105 "Failed to register decode complete callback, return code: " 106 "%d\n", 107 register_result); 108 return false; 109 } 110 // Init the encoder and decoder 111 uint32_t nbr_of_cores = 1; 112 if (!config_.use_single_core) { 113 nbr_of_cores = CpuInfo::DetectNumberOfCores(); 114 } 115 int32_t init_result = 116 encoder_->InitEncode(config_.codec_settings, nbr_of_cores, 117 config_.networking_config.max_payload_size_in_bytes); 118 if (init_result != WEBRTC_VIDEO_CODEC_OK) { 119 fprintf(stderr, "Failed to initialize VideoEncoder, return code: %d\n", 120 init_result); 121 return false; 122 } 123 init_result = decoder_->InitDecode(config_.codec_settings, nbr_of_cores); 124 if (init_result != WEBRTC_VIDEO_CODEC_OK) { 125 fprintf(stderr, "Failed to initialize VideoDecoder, return code: %d\n", 126 init_result); 127 return false; 128 } 129 130 if (config_.verbose) { 131 printf("Video Processor:\n"); 132 printf(" #CPU cores used : %d\n", nbr_of_cores); 133 printf(" Total # of frames: %d\n", frame_reader_->NumberOfFrames()); 134 printf(" Codec settings:\n"); 135 printf(" Start bitrate : %d kbps\n", 136 config_.codec_settings->startBitrate); 137 printf(" Width : %d\n", config_.codec_settings->width); 138 printf(" Height : %d\n", config_.codec_settings->height); 139 } 140 initialized_ = true; 141 return true; 142 } 143 144 VideoProcessorImpl::~VideoProcessorImpl() { 145 delete[] source_buffer_; 146 delete[] last_successful_frame_buffer_; 147 encoder_->RegisterEncodeCompleteCallback(NULL); 148 delete encode_callback_; 149 decoder_->RegisterDecodeCompleteCallback(NULL); 150 delete decode_callback_; 151 } 152 153 void VideoProcessorImpl::SetRates(int bit_rate, int frame_rate) { 154 int set_rates_result = encoder_->SetRates(bit_rate, frame_rate); 155 assert(set_rates_result >= 0); 156 if (set_rates_result < 0) { 157 fprintf(stderr, 158 "Failed to update encoder with new rate %d, " 159 "return code: %d\n", 160 bit_rate, set_rates_result); 161 } 162 num_dropped_frames_ = 0; 163 num_spatial_resizes_ = 0; 164 } 165 166 size_t VideoProcessorImpl::EncodedFrameSize() { 167 return encoded_frame_size_; 168 } 169 170 FrameType VideoProcessorImpl::EncodedFrameType() { 171 return encoded_frame_type_; 172 } 173 174 int VideoProcessorImpl::NumberDroppedFrames() { 175 return num_dropped_frames_; 176 } 177 178 int VideoProcessorImpl::NumberSpatialResizes() { 179 return num_spatial_resizes_; 180 } 181 182 bool VideoProcessorImpl::ProcessFrame(int frame_number) { 183 assert(frame_number >= 0); 184 if (!initialized_) { 185 fprintf(stderr, "Attempting to use uninitialized VideoProcessor!\n"); 186 return false; 187 } 188 // |prev_time_stamp_| is used for getting number of dropped frames. 189 if (frame_number == 0) { 190 prev_time_stamp_ = -1; 191 } 192 if (frame_reader_->ReadFrame(source_buffer_)) { 193 // Copy the source frame to the newly read frame data. 194 source_frame_.CreateFrame(source_buffer_, config_.codec_settings->width, 195 config_.codec_settings->height, kVideoRotation_0); 196 197 // Ensure we have a new statistics data object we can fill: 198 FrameStatistic& stat = stats_->NewFrame(frame_number); 199 200 encode_start_ = TickTime::Now(); 201 // Use the frame number as "timestamp" to identify frames 202 source_frame_.set_timestamp(frame_number); 203 204 // Decide if we're going to force a keyframe: 205 std::vector<FrameType> frame_types(1, kVideoFrameDelta); 206 if (config_.keyframe_interval > 0 && 207 frame_number % config_.keyframe_interval == 0) { 208 frame_types[0] = kVideoFrameKey; 209 } 210 211 // For dropped frames, we regard them as zero size encoded frames. 212 encoded_frame_size_ = 0; 213 encoded_frame_type_ = kVideoFrameDelta; 214 215 int32_t encode_result = encoder_->Encode(source_frame_, NULL, &frame_types); 216 217 if (encode_result != WEBRTC_VIDEO_CODEC_OK) { 218 fprintf(stderr, "Failed to encode frame %d, return code: %d\n", 219 frame_number, encode_result); 220 } 221 stat.encode_return_code = encode_result; 222 return true; 223 } else { 224 return false; // we've reached the last frame 225 } 226 } 227 228 void VideoProcessorImpl::FrameEncoded(const EncodedImage& encoded_image) { 229 // Timestamp is frame number, so this gives us #dropped frames. 230 int num_dropped_from_prev_encode = 231 encoded_image._timeStamp - prev_time_stamp_ - 1; 232 num_dropped_frames_ += num_dropped_from_prev_encode; 233 prev_time_stamp_ = encoded_image._timeStamp; 234 if (num_dropped_from_prev_encode > 0) { 235 // For dropped frames, we write out the last decoded frame to avoid getting 236 // out of sync for the computation of PSNR and SSIM. 237 for (int i = 0; i < num_dropped_from_prev_encode; i++) { 238 frame_writer_->WriteFrame(last_successful_frame_buffer_); 239 } 240 } 241 // Frame is not dropped, so update the encoded frame size 242 // (encoder callback is only called for non-zero length frames). 243 encoded_frame_size_ = encoded_image._length; 244 245 encoded_frame_type_ = encoded_image._frameType; 246 247 TickTime encode_stop = TickTime::Now(); 248 int frame_number = encoded_image._timeStamp; 249 FrameStatistic& stat = stats_->stats_[frame_number]; 250 stat.encode_time_in_us = 251 GetElapsedTimeMicroseconds(encode_start_, encode_stop); 252 stat.encoding_successful = true; 253 stat.encoded_frame_length_in_bytes = encoded_image._length; 254 stat.frame_number = encoded_image._timeStamp; 255 stat.frame_type = encoded_image._frameType; 256 stat.bit_rate_in_kbps = encoded_image._length * bit_rate_factor_; 257 stat.total_packets = 258 encoded_image._length / config_.networking_config.packet_size_in_bytes + 259 1; 260 261 // Perform packet loss if criteria is fullfilled: 262 bool exclude_this_frame = false; 263 // Only keyframes can be excluded 264 if (encoded_image._frameType == kVideoFrameKey) { 265 switch (config_.exclude_frame_types) { 266 case kExcludeOnlyFirstKeyFrame: 267 if (!first_key_frame_has_been_excluded_) { 268 first_key_frame_has_been_excluded_ = true; 269 exclude_this_frame = true; 270 } 271 break; 272 case kExcludeAllKeyFrames: 273 exclude_this_frame = true; 274 break; 275 default: 276 assert(false); 277 } 278 } 279 rtc::scoped_ptr<uint8_t[]> copied_buffer(new uint8_t[encoded_image._length]); 280 memcpy(copied_buffer.get(), encoded_image._buffer, encoded_image._length); 281 EncodedImage copied_image; 282 memcpy(&copied_image, &encoded_image, sizeof(copied_image)); 283 copied_image._size = copied_image._length; 284 copied_image._buffer = copied_buffer.get(); 285 if (!exclude_this_frame) { 286 stat.packets_dropped = 287 packet_manipulator_->ManipulatePackets(&copied_image); 288 } 289 290 // Keep track of if frames are lost due to packet loss so we can tell 291 // this to the encoder (this is handled by the RTP logic in the full stack) 292 decode_start_ = TickTime::Now(); 293 // TODO(kjellander): Pass fragmentation header to the decoder when 294 // CL 172001 has been submitted and PacketManipulator supports this. 295 int32_t decode_result = 296 decoder_->Decode(copied_image, last_frame_missing_, NULL); 297 stat.decode_return_code = decode_result; 298 if (decode_result != WEBRTC_VIDEO_CODEC_OK) { 299 // Write the last successful frame the output file to avoid getting it out 300 // of sync with the source file for SSIM and PSNR comparisons: 301 frame_writer_->WriteFrame(last_successful_frame_buffer_); 302 } 303 // save status for losses so we can inform the decoder for the next frame: 304 last_frame_missing_ = copied_image._length == 0; 305 } 306 307 void VideoProcessorImpl::FrameDecoded(const VideoFrame& image) { 308 TickTime decode_stop = TickTime::Now(); 309 int frame_number = image.timestamp(); 310 // Report stats 311 FrameStatistic& stat = stats_->stats_[frame_number]; 312 stat.decode_time_in_us = 313 GetElapsedTimeMicroseconds(decode_start_, decode_stop); 314 stat.decoding_successful = true; 315 316 // Check for resize action (either down or up): 317 if (static_cast<int>(image.width()) != last_encoder_frame_width_ || 318 static_cast<int>(image.height()) != last_encoder_frame_height_) { 319 ++num_spatial_resizes_; 320 last_encoder_frame_width_ = image.width(); 321 last_encoder_frame_height_ = image.height(); 322 } 323 // Check if codec size is different from native/original size, and if so, 324 // upsample back to original size: needed for PSNR and SSIM computations. 325 if (image.width() != config_.codec_settings->width || 326 image.height() != config_.codec_settings->height) { 327 VideoFrame up_image; 328 int ret_val = scaler_.Set( 329 image.width(), image.height(), config_.codec_settings->width, 330 config_.codec_settings->height, kI420, kI420, kScaleBilinear); 331 assert(ret_val >= 0); 332 if (ret_val < 0) { 333 fprintf(stderr, "Failed to set scalar for frame: %d, return code: %d\n", 334 frame_number, ret_val); 335 } 336 ret_val = scaler_.Scale(image, &up_image); 337 assert(ret_val >= 0); 338 if (ret_val < 0) { 339 fprintf(stderr, "Failed to scale frame: %d, return code: %d\n", 340 frame_number, ret_val); 341 } 342 // TODO(mikhal): Extracting the buffer for now - need to update test. 343 size_t length = CalcBufferSize(kI420, up_image.width(), up_image.height()); 344 rtc::scoped_ptr<uint8_t[]> image_buffer(new uint8_t[length]); 345 int extracted_length = ExtractBuffer(up_image, length, image_buffer.get()); 346 assert(extracted_length > 0); 347 // Update our copy of the last successful frame: 348 memcpy(last_successful_frame_buffer_, image_buffer.get(), extracted_length); 349 bool write_success = frame_writer_->WriteFrame(image_buffer.get()); 350 assert(write_success); 351 if (!write_success) { 352 fprintf(stderr, "Failed to write frame %d to disk!", frame_number); 353 } 354 } else { // No resize. 355 // Update our copy of the last successful frame: 356 // TODO(mikhal): Add as a member function, so won't be allocated per frame. 357 size_t length = CalcBufferSize(kI420, image.width(), image.height()); 358 rtc::scoped_ptr<uint8_t[]> image_buffer(new uint8_t[length]); 359 int extracted_length = ExtractBuffer(image, length, image_buffer.get()); 360 assert(extracted_length > 0); 361 memcpy(last_successful_frame_buffer_, image_buffer.get(), extracted_length); 362 363 bool write_success = frame_writer_->WriteFrame(image_buffer.get()); 364 assert(write_success); 365 if (!write_success) { 366 fprintf(stderr, "Failed to write frame %d to disk!", frame_number); 367 } 368 } 369 } 370 371 int VideoProcessorImpl::GetElapsedTimeMicroseconds( 372 const webrtc::TickTime& start, 373 const webrtc::TickTime& stop) { 374 uint64_t encode_time = (stop - start).Microseconds(); 375 assert(encode_time < 376 static_cast<unsigned int>(std::numeric_limits<int>::max())); 377 return static_cast<int>(encode_time); 378 } 379 380 const char* ExcludeFrameTypesToStr(ExcludeFrameTypes e) { 381 switch (e) { 382 case kExcludeOnlyFirstKeyFrame: 383 return "ExcludeOnlyFirstKeyFrame"; 384 case kExcludeAllKeyFrames: 385 return "ExcludeAllKeyFrames"; 386 default: 387 assert(false); 388 return "Unknown"; 389 } 390 } 391 392 const char* VideoCodecTypeToStr(webrtc::VideoCodecType e) { 393 switch (e) { 394 case kVideoCodecVP8: 395 return "VP8"; 396 case kVideoCodecI420: 397 return "I420"; 398 case kVideoCodecRED: 399 return "RED"; 400 case kVideoCodecULPFEC: 401 return "ULPFEC"; 402 case kVideoCodecUnknown: 403 return "Unknown"; 404 default: 405 assert(false); 406 return "Unknown"; 407 } 408 } 409 410 // Callbacks 411 int32_t VideoProcessorImpl::VideoProcessorEncodeCompleteCallback::Encoded( 412 const EncodedImage& encoded_image, 413 const webrtc::CodecSpecificInfo* codec_specific_info, 414 const webrtc::RTPFragmentationHeader* fragmentation) { 415 video_processor_->FrameEncoded(encoded_image); // Forward to parent class. 416 return 0; 417 } 418 int32_t VideoProcessorImpl::VideoProcessorDecodeCompleteCallback::Decoded( 419 VideoFrame& image) { 420 video_processor_->FrameDecoded(image); // forward to parent class 421 return 0; 422 } 423 424 } // namespace test 425 } // namespace webrtc 426
