1 // Copyright 2014 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "media/cast/sender/vp8_encoder.h" 6 7 #include <vector> 8 9 #include "base/logging.h" 10 #include "media/base/video_frame.h" 11 #include "media/cast/cast_defines.h" 12 #include "media/cast/net/cast_transport_config.h" 13 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h" 14 15 namespace media { 16 namespace cast { 17 18 static const uint32 kMinIntra = 300; 19 20 Vp8Encoder::Vp8Encoder(const VideoSenderConfig& video_config, 21 int max_unacked_frames) 22 : cast_config_(video_config), 23 use_multiple_video_buffers_( 24 cast_config_.max_number_of_video_buffers_used == 25 kNumberOfVp8VideoBuffers), 26 key_frame_requested_(true), 27 first_frame_received_(false), 28 last_encoded_frame_id_(kStartFrameId), 29 last_acked_frame_id_(kStartFrameId), 30 frame_id_to_reference_(kStartFrameId - 1), 31 undroppable_frames_(0) { 32 // VP8 have 3 buffers available for prediction, with 33 // max_number_of_video_buffers_used set to 1 we maximize the coding efficiency 34 // however in this mode we can not skip frames in the receiver to catch up 35 // after a temporary network outage; with max_number_of_video_buffers_used 36 // set to 3 we allow 2 frames to be skipped by the receiver without error 37 // propagation. 38 DCHECK(cast_config_.max_number_of_video_buffers_used == 1 || 39 cast_config_.max_number_of_video_buffers_used == 40 kNumberOfVp8VideoBuffers) 41 << "Invalid argument"; 42 43 thread_checker_.DetachFromThread(); 44 } 45 46 Vp8Encoder::~Vp8Encoder() { 47 vpx_codec_destroy(encoder_.get()); 48 vpx_img_free(raw_image_); 49 } 50 51 void Vp8Encoder::Initialize() { 52 DCHECK(thread_checker_.CalledOnValidThread()); 53 config_.reset(new vpx_codec_enc_cfg_t()); 54 encoder_.reset(new vpx_codec_ctx_t()); 55 56 // Creating a wrapper to the image - setting image data to NULL. Actual 57 // pointer will be set during encode. Setting align to 1, as it is 58 // meaningless (actual memory is not allocated). 59 raw_image_ = vpx_img_wrap( 60 NULL, VPX_IMG_FMT_I420, cast_config_.width, cast_config_.height, 1, NULL); 61 62 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { 63 buffer_state_[i].frame_id = kStartFrameId; 64 buffer_state_[i].state = kBufferStartState; 65 } 66 InitEncode(cast_config_.number_of_encode_threads); 67 } 68 69 void Vp8Encoder::InitEncode(int number_of_encode_threads) { 70 DCHECK(thread_checker_.CalledOnValidThread()); 71 // Populate encoder configuration with default values. 72 if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), config_.get(), 0)) { 73 DCHECK(false) << "Invalid return value"; 74 } 75 config_->g_w = cast_config_.width; 76 config_->g_h = cast_config_.height; 77 config_->rc_target_bitrate = cast_config_.start_bitrate / 1000; // In kbit/s. 78 79 // Setting the codec time base. 80 config_->g_timebase.num = 1; 81 config_->g_timebase.den = kVideoFrequency; 82 config_->g_lag_in_frames = 0; 83 config_->kf_mode = VPX_KF_DISABLED; 84 if (use_multiple_video_buffers_) { 85 // We must enable error resilience when we use multiple buffers, due to 86 // codec requirements. 87 config_->g_error_resilient = 1; 88 } 89 config_->g_threads = number_of_encode_threads; 90 91 // Rate control settings. 92 // Never allow the encoder to drop frame internally. 93 config_->rc_dropframe_thresh = 0; 94 config_->rc_end_usage = VPX_CBR; 95 config_->g_pass = VPX_RC_ONE_PASS; 96 config_->rc_resize_allowed = 0; 97 config_->rc_min_quantizer = cast_config_.min_qp; 98 config_->rc_max_quantizer = cast_config_.max_qp; 99 config_->rc_undershoot_pct = 100; 100 config_->rc_overshoot_pct = 15; 101 config_->rc_buf_initial_sz = 500; 102 config_->rc_buf_optimal_sz = 600; 103 config_->rc_buf_sz = 1000; 104 105 // set the maximum target size of any key-frame. 106 uint32 rc_max_intra_target = MaxIntraTarget(config_->rc_buf_optimal_sz); 107 vpx_codec_flags_t flags = 0; 108 if (vpx_codec_enc_init( 109 encoder_.get(), vpx_codec_vp8_cx(), config_.get(), flags)) { 110 DCHECK(false) << "vpx_codec_enc_init() failed."; 111 encoder_.reset(); 112 return; 113 } 114 vpx_codec_control(encoder_.get(), VP8E_SET_STATIC_THRESHOLD, 1); 115 vpx_codec_control(encoder_.get(), VP8E_SET_NOISE_SENSITIVITY, 0); 116 vpx_codec_control(encoder_.get(), VP8E_SET_CPUUSED, -6); 117 vpx_codec_control( 118 encoder_.get(), VP8E_SET_MAX_INTRA_BITRATE_PCT, rc_max_intra_target); 119 } 120 121 bool Vp8Encoder::Encode(const scoped_refptr<media::VideoFrame>& video_frame, 122 EncodedFrame* encoded_image) { 123 DCHECK(thread_checker_.CalledOnValidThread()); 124 // Image in vpx_image_t format. 125 // Input image is const. VP8's raw image is not defined as const. 126 raw_image_->planes[VPX_PLANE_Y] = 127 const_cast<uint8*>(video_frame->data(VideoFrame::kYPlane)); 128 raw_image_->planes[VPX_PLANE_U] = 129 const_cast<uint8*>(video_frame->data(VideoFrame::kUPlane)); 130 raw_image_->planes[VPX_PLANE_V] = 131 const_cast<uint8*>(video_frame->data(VideoFrame::kVPlane)); 132 133 raw_image_->stride[VPX_PLANE_Y] = video_frame->stride(VideoFrame::kYPlane); 134 raw_image_->stride[VPX_PLANE_U] = video_frame->stride(VideoFrame::kUPlane); 135 raw_image_->stride[VPX_PLANE_V] = video_frame->stride(VideoFrame::kVPlane); 136 137 uint32 latest_frame_id_to_reference; 138 Vp8Buffers buffer_to_update; 139 vpx_codec_flags_t flags = 0; 140 if (key_frame_requested_) { 141 flags = VPX_EFLAG_FORCE_KF; 142 // Self reference. 143 latest_frame_id_to_reference = last_encoded_frame_id_ + 1; 144 // We can pick any buffer as buffer_to_update since we update 145 // them all. 146 buffer_to_update = kLastBuffer; 147 } else { 148 // Reference all acked frames (buffers). 149 latest_frame_id_to_reference = GetCodecReferenceFlags(&flags); 150 buffer_to_update = GetNextBufferToUpdate(); 151 GetCodecUpdateFlags(buffer_to_update, &flags); 152 } 153 154 // Note: The duration does not reflect the real time between frames. This is 155 // done to keep the encoder happy. 156 // 157 // TODO(miu): This is a semi-hack. We should consider using 158 // |video_frame->timestamp()| instead. 159 uint32 duration = kVideoFrequency / cast_config_.max_frame_rate; 160 161 // Note: Timestamp here is used for bitrate calculation. The absolute value 162 // is not important. 163 if (!first_frame_received_) { 164 first_frame_received_ = true; 165 first_frame_timestamp_ = video_frame->timestamp(); 166 } 167 168 vpx_codec_pts_t timestamp = 169 (video_frame->timestamp() - first_frame_timestamp_).InMicroseconds() * 170 kVideoFrequency / base::Time::kMicrosecondsPerSecond; 171 172 if (vpx_codec_encode(encoder_.get(), 173 raw_image_, 174 timestamp, 175 duration, 176 flags, 177 VPX_DL_REALTIME) != VPX_CODEC_OK) { 178 LOG(ERROR) << "Failed to encode for once."; 179 return false; 180 } 181 182 // Get encoded frame. 183 const vpx_codec_cx_pkt_t* pkt = NULL; 184 vpx_codec_iter_t iter = NULL; 185 bool is_key_frame = false; 186 while ((pkt = vpx_codec_get_cx_data(encoder_.get(), &iter)) != NULL) { 187 if (pkt->kind != VPX_CODEC_CX_FRAME_PKT) 188 continue; 189 encoded_image->data.assign( 190 static_cast<const uint8*>(pkt->data.frame.buf), 191 static_cast<const uint8*>(pkt->data.frame.buf) + pkt->data.frame.sz); 192 is_key_frame = !!(pkt->data.frame.flags & VPX_FRAME_IS_KEY); 193 break; // Done, since all data is provided in one CX_FRAME_PKT packet. 194 } 195 // Don't update frame_id for zero size frames. 196 if (encoded_image->data.empty()) 197 return true; 198 199 // Populate the encoded frame. 200 encoded_image->frame_id = ++last_encoded_frame_id_; 201 if (is_key_frame) { 202 // TODO(Hubbe): Replace "dependency" with a "bool is_key_frame". 203 encoded_image->dependency = EncodedFrame::KEY; 204 encoded_image->referenced_frame_id = encoded_image->frame_id; 205 } else { 206 encoded_image->dependency = EncodedFrame::DEPENDENT; 207 encoded_image->referenced_frame_id = latest_frame_id_to_reference; 208 } 209 210 DVLOG(1) << "VP8 encoded frame_id " << encoded_image->frame_id 211 << ", sized:" << encoded_image->data.size(); 212 213 if (is_key_frame) { 214 key_frame_requested_ = false; 215 216 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { 217 buffer_state_[i].state = kBufferSent; 218 buffer_state_[i].frame_id = encoded_image->frame_id; 219 } 220 } else { 221 if (buffer_to_update != kNoBuffer) { 222 buffer_state_[buffer_to_update].state = kBufferSent; 223 buffer_state_[buffer_to_update].frame_id = encoded_image->frame_id; 224 } 225 } 226 return true; 227 } 228 229 uint32 Vp8Encoder::GetCodecReferenceFlags(vpx_codec_flags_t* flags) { 230 if (!use_multiple_video_buffers_) 231 return last_encoded_frame_id_; 232 233 const uint32 kMagicFrameOffset = 512; 234 // We set latest_frame_to_reference to an old frame so that 235 // IsNewerFrameId will work correctly. 236 uint32 latest_frame_to_reference = 237 last_encoded_frame_id_ - kMagicFrameOffset; 238 239 // Reference all acked frames. 240 // TODO(hubbe): We may also want to allow references to the 241 // last encoded frame, if that frame was assigned to a buffer. 242 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { 243 if (buffer_state_[i].state == kBufferAcked) { 244 if (IsNewerFrameId(buffer_state_[i].frame_id, 245 latest_frame_to_reference)) { 246 latest_frame_to_reference = buffer_state_[i].frame_id; 247 } 248 } else { 249 switch (i) { 250 case kAltRefBuffer: 251 *flags |= VP8_EFLAG_NO_REF_ARF; 252 break; 253 case kGoldenBuffer: 254 *flags |= VP8_EFLAG_NO_REF_GF; 255 break; 256 case kLastBuffer: 257 *flags |= VP8_EFLAG_NO_REF_LAST; 258 break; 259 } 260 } 261 } 262 263 if (latest_frame_to_reference == 264 last_encoded_frame_id_ - kMagicFrameOffset) { 265 // We have nothing to reference, it's kind of like a key frame, 266 // but doesn't reset buffers. 267 latest_frame_to_reference = last_encoded_frame_id_ + 1; 268 } 269 270 return latest_frame_to_reference; 271 } 272 273 Vp8Encoder::Vp8Buffers Vp8Encoder::GetNextBufferToUpdate() { 274 if (!use_multiple_video_buffers_) 275 return kNoBuffer; 276 277 // The goal here is to make sure that we always keep one ACKed 278 // buffer while trying to get an ACK for a newer buffer as we go. 279 // Here are the rules for which buffer to select for update: 280 // 1. If there is a buffer in state kStartState, use it. 281 // 2. If there is a buffer other than the oldest buffer 282 // which is Acked, use the oldest buffer. 283 // 3. If there are Sent buffers which are older than 284 // latest_acked_frame_, use the oldest one. 285 // 4. If all else fails, just overwrite the newest buffer, 286 // but no more than 3 times in a row. 287 // TODO(hubbe): Figure out if 3 is optimal. 288 // Note, rule 1-3 describe cases where there is a "free" buffer 289 // that we can use. Rule 4 describes what happens when there is 290 // no free buffer available. 291 292 // Buffers, sorted from oldest frame to newest. 293 Vp8Encoder::Vp8Buffers buffers[kNumberOfVp8VideoBuffers]; 294 295 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { 296 Vp8Encoder::Vp8Buffers buffer = static_cast<Vp8Encoder::Vp8Buffers>(i); 297 298 // Rule 1 299 if (buffer_state_[buffer].state == kBufferStartState) { 300 undroppable_frames_ = 0; 301 return buffer; 302 } 303 buffers[buffer] = buffer; 304 } 305 306 // Sorting three elements with selection sort. 307 for (int i = 0; i < kNumberOfVp8VideoBuffers - 1; i++) { 308 for (int j = i + 1; j < kNumberOfVp8VideoBuffers; j++) { 309 if (IsOlderFrameId(buffer_state_[buffers[j]].frame_id, 310 buffer_state_[buffers[i]].frame_id)) { 311 std::swap(buffers[i], buffers[j]); 312 } 313 } 314 } 315 316 // Rule 2 317 if (buffer_state_[buffers[1]].state == kBufferAcked || 318 buffer_state_[buffers[2]].state == kBufferAcked) { 319 undroppable_frames_ = 0; 320 return buffers[0]; 321 } 322 323 // Rule 3 324 for (int i = 0; i < kNumberOfVp8VideoBuffers; i++) { 325 if (buffer_state_[buffers[i]].state == kBufferSent && 326 IsOlderFrameId(buffer_state_[buffers[i]].frame_id, 327 last_acked_frame_id_)) { 328 undroppable_frames_ = 0; 329 return buffers[i]; 330 } 331 } 332 333 // Rule 4 334 if (undroppable_frames_ >= 3) { 335 undroppable_frames_ = 0; 336 return kNoBuffer; 337 } else { 338 undroppable_frames_++; 339 return buffers[kNumberOfVp8VideoBuffers - 1]; 340 } 341 } 342 343 void Vp8Encoder::GetCodecUpdateFlags(Vp8Buffers buffer_to_update, 344 vpx_codec_flags_t* flags) { 345 if (!use_multiple_video_buffers_) 346 return; 347 348 // Update at most one buffer, except for key-frames. 349 switch (buffer_to_update) { 350 case kAltRefBuffer: 351 *flags |= VP8_EFLAG_NO_UPD_GF; 352 *flags |= VP8_EFLAG_NO_UPD_LAST; 353 break; 354 case kLastBuffer: 355 *flags |= VP8_EFLAG_NO_UPD_GF; 356 *flags |= VP8_EFLAG_NO_UPD_ARF; 357 break; 358 case kGoldenBuffer: 359 *flags |= VP8_EFLAG_NO_UPD_ARF; 360 *flags |= VP8_EFLAG_NO_UPD_LAST; 361 break; 362 case kNoBuffer: 363 *flags |= VP8_EFLAG_NO_UPD_ARF; 364 *flags |= VP8_EFLAG_NO_UPD_GF; 365 *flags |= VP8_EFLAG_NO_UPD_LAST; 366 *flags |= VP8_EFLAG_NO_UPD_ENTROPY; 367 break; 368 } 369 } 370 371 void Vp8Encoder::UpdateRates(uint32 new_bitrate) { 372 DCHECK(thread_checker_.CalledOnValidThread()); 373 uint32 new_bitrate_kbit = new_bitrate / 1000; 374 if (config_->rc_target_bitrate == new_bitrate_kbit) 375 return; 376 377 config_->rc_target_bitrate = new_bitrate_kbit; 378 379 // Update encoder context. 380 if (vpx_codec_enc_config_set(encoder_.get(), config_.get())) { 381 DCHECK(false) << "Invalid return value"; 382 } 383 } 384 385 void Vp8Encoder::LatestFrameIdToReference(uint32 frame_id) { 386 DCHECK(thread_checker_.CalledOnValidThread()); 387 if (!use_multiple_video_buffers_) 388 return; 389 390 VLOG(1) << "VP8 ok to reference frame:" << static_cast<int>(frame_id); 391 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { 392 if (frame_id == buffer_state_[i].frame_id) { 393 buffer_state_[i].state = kBufferAcked; 394 break; 395 } 396 } 397 if (IsOlderFrameId(last_acked_frame_id_, frame_id)) { 398 last_acked_frame_id_ = frame_id; 399 } 400 } 401 402 void Vp8Encoder::GenerateKeyFrame() { 403 DCHECK(thread_checker_.CalledOnValidThread()); 404 key_frame_requested_ = true; 405 } 406 407 // Calculate the max size of the key frame relative to a normal delta frame. 408 uint32 Vp8Encoder::MaxIntraTarget(uint32 optimal_buffer_size_ms) const { 409 // Set max to the optimal buffer level (normalized by target BR), 410 // and scaled by a scale_parameter. 411 // Max target size = scalePar * optimalBufferSize * targetBR[Kbps]. 412 // This values is presented in percentage of perFrameBw: 413 // perFrameBw = targetBR[Kbps] * 1000 / frameRate. 414 // The target in % is as follows: 415 416 float scale_parameter = 0.5; 417 uint32 target_pct = optimal_buffer_size_ms * scale_parameter * 418 cast_config_.max_frame_rate / 10; 419 420 // Don't go below 3 times the per frame bandwidth. 421 return std::max(target_pct, kMinIntra); 422 } 423 424 } // namespace cast 425 } // namespace media 426