1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 // 5 // TODO (pwestin): add a link to the design document describing the generic 6 // protocol and the VP8 specific details. 7 #include "media/cast/video_sender/codecs/vp8/vp8_encoder.h" 8 9 #include <vector> 10 11 #include "base/logging.h" 12 #include "media/base/video_frame.h" 13 #include "media/cast/cast_defines.h" 14 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h" 15 16 namespace media { 17 namespace cast { 18 19 static const uint32 kMinIntra = 300; 20 21 Vp8Encoder::Vp8Encoder(const VideoSenderConfig& video_config, 22 uint8 max_unacked_frames) 23 : cast_config_(video_config), 24 use_multiple_video_buffers_( 25 cast_config_.max_number_of_video_buffers_used == 26 kNumberOfVp8VideoBuffers), 27 max_number_of_repeated_buffers_in_a_row_( 28 (max_unacked_frames > kNumberOfVp8VideoBuffers) ? 29 ((max_unacked_frames - 1) / kNumberOfVp8VideoBuffers) : 0), 30 config_(new vpx_codec_enc_cfg_t()), 31 encoder_(new vpx_codec_ctx_t()), 32 // Creating a wrapper to the image - setting image data to NULL. Actual 33 // pointer will be set during encode. Setting align to 1, as it is 34 // meaningless (actual memory is not allocated). 35 raw_image_(vpx_img_wrap(NULL, IMG_FMT_I420, video_config.width, 36 video_config.height, 1, NULL)), 37 key_frame_requested_(true), 38 timestamp_(0), 39 last_encoded_frame_id_(kStartFrameId), 40 number_of_repeated_buffers_(0) { 41 // TODO(pwestin): we need to figure out how to synchronize the acking with the 42 // internal state of the encoder, ideally the encoder will tell if we can 43 // send another frame. 44 DCHECK(!use_multiple_video_buffers_ || 45 max_number_of_repeated_buffers_in_a_row_ == 0) << "Invalid config"; 46 47 // VP8 have 3 buffers available for prediction, with 48 // max_number_of_video_buffers_used set to 1 we maximize the coding efficiency 49 // however in this mode we can not skip frames in the receiver to catch up 50 // after a temporary network outage; with max_number_of_video_buffers_used 51 // set to 3 we allow 2 frames to be skipped by the receiver without error 52 // propagation. 53 DCHECK(cast_config_.max_number_of_video_buffers_used == 1 || 54 cast_config_.max_number_of_video_buffers_used == 55 kNumberOfVp8VideoBuffers) << "Invalid argument"; 56 57 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { 58 acked_frame_buffers_[i] = true; 59 used_buffers_frame_id_[i] = kStartFrameId; 60 } 61 InitEncode(video_config.number_of_cores); 62 } 63 64 Vp8Encoder::~Vp8Encoder() { 65 vpx_codec_destroy(encoder_); 66 vpx_img_free(raw_image_); 67 } 68 69 void Vp8Encoder::InitEncode(int number_of_cores) { 70 // Populate encoder configuration with default values. 71 if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), config_.get(), 0)) { 72 DCHECK(false) << "Invalid return value"; 73 } 74 config_->g_w = cast_config_.width; 75 config_->g_h = cast_config_.height; 76 config_->rc_target_bitrate = cast_config_.start_bitrate / 1000; // In kbit/s. 77 78 // Setting the codec time base. 79 config_->g_timebase.num = 1; 80 config_->g_timebase.den = kVideoFrequency; 81 config_->g_lag_in_frames = 0; 82 config_->kf_mode = VPX_KF_DISABLED; 83 if (use_multiple_video_buffers_) { 84 // We must enable error resilience when we use multiple buffers, due to 85 // codec requirements. 86 config_->g_error_resilient = 1; 87 } 88 89 if (cast_config_.width * cast_config_.height > 640 * 480 90 && number_of_cores >= 2) { 91 config_->g_threads = 2; // 2 threads for qHD/HD. 92 } else { 93 config_->g_threads = 1; // 1 thread for VGA or less. 94 } 95 96 // Rate control settings. 97 // TODO(pwestin): revisit these constants. Currently identical to webrtc. 98 config_->rc_dropframe_thresh = 30; 99 config_->rc_end_usage = VPX_CBR; 100 config_->g_pass = VPX_RC_ONE_PASS; 101 config_->rc_resize_allowed = 0; 102 config_->rc_min_quantizer = cast_config_.min_qp; 103 config_->rc_max_quantizer = cast_config_.max_qp; 104 config_->rc_undershoot_pct = 100; 105 config_->rc_overshoot_pct = 15; 106 config_->rc_buf_initial_sz = 500; 107 config_->rc_buf_optimal_sz = 600; 108 config_->rc_buf_sz = 1000; 109 110 // set the maximum target size of any key-frame. 111 uint32 rc_max_intra_target = MaxIntraTarget(config_->rc_buf_optimal_sz); 112 vpx_codec_flags_t flags = 0; 113 // TODO(mikhal): Tune settings. 114 if (vpx_codec_enc_init(encoder_, vpx_codec_vp8_cx(), config_.get(), flags)) { 115 DCHECK(false) << "Invalid return value"; 116 } 117 vpx_codec_control(encoder_, VP8E_SET_STATIC_THRESHOLD, 1); 118 vpx_codec_control(encoder_, VP8E_SET_NOISE_SENSITIVITY, 0); 119 vpx_codec_control(encoder_, VP8E_SET_CPUUSED, -6); 120 vpx_codec_control(encoder_, VP8E_SET_MAX_INTRA_BITRATE_PCT, 121 rc_max_intra_target); 122 } 123 124 bool Vp8Encoder::Encode(const scoped_refptr<media::VideoFrame>& video_frame, 125 EncodedVideoFrame* encoded_image) { 126 // Image in vpx_image_t format. 127 // Input image is const. VP8's raw image is not defined as const. 128 raw_image_->planes[PLANE_Y] = 129 const_cast<uint8*>(video_frame->data(VideoFrame::kYPlane)); 130 raw_image_->planes[PLANE_U] = 131 const_cast<uint8*>(video_frame->data(VideoFrame::kUPlane)); 132 raw_image_->planes[PLANE_V] = 133 const_cast<uint8*>(video_frame->data(VideoFrame::kVPlane)); 134 135 raw_image_->stride[VPX_PLANE_Y] = video_frame->stride(VideoFrame::kYPlane); 136 raw_image_->stride[VPX_PLANE_U] = video_frame->stride(VideoFrame::kUPlane); 137 raw_image_->stride[VPX_PLANE_V] = video_frame->stride(VideoFrame::kVPlane); 138 139 uint8 latest_frame_id_to_reference; 140 Vp8Buffers buffer_to_update; 141 vpx_codec_flags_t flags = 0; 142 if (key_frame_requested_) { 143 flags = VPX_EFLAG_FORCE_KF; 144 // Self reference. 145 latest_frame_id_to_reference = 146 static_cast<uint8>(last_encoded_frame_id_ + 1); 147 // We can pick any buffer as buffer_to_update since we update 148 // them all. 149 buffer_to_update = kLastBuffer; 150 } else { 151 // Reference all acked frames (buffers). 152 latest_frame_id_to_reference = GetLatestFrameIdToReference(); 153 GetCodecReferenceFlags(&flags); 154 buffer_to_update = GetNextBufferToUpdate(); 155 GetCodecUpdateFlags(buffer_to_update, &flags); 156 } 157 158 // Note: The duration does not reflect the real time between frames. This is 159 // done to keep the encoder happy. 160 uint32 duration = kVideoFrequency / cast_config_.max_frame_rate; 161 if (vpx_codec_encode(encoder_, raw_image_, timestamp_, duration, flags, 162 VPX_DL_REALTIME)) { 163 return false; 164 } 165 timestamp_ += duration; 166 167 // Get encoded frame. 168 const vpx_codec_cx_pkt_t *pkt = NULL; 169 vpx_codec_iter_t iter = NULL; 170 size_t total_size = 0; 171 while ((pkt = vpx_codec_get_cx_data(encoder_, &iter)) != NULL) { 172 if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { 173 total_size += pkt->data.frame.sz; 174 encoded_image->data.reserve(total_size); 175 encoded_image->data.insert( 176 encoded_image->data.end(), 177 static_cast<const uint8*>(pkt->data.frame.buf), 178 static_cast<const uint8*>(pkt->data.frame.buf) + 179 pkt->data.frame.sz); 180 if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) { 181 encoded_image->key_frame = true; 182 } else { 183 encoded_image->key_frame = false; 184 } 185 } 186 } 187 // Don't update frame_id for zero size frames. 188 if (total_size == 0) return true; 189 190 // Populate the encoded frame. 191 encoded_image->codec = kVp8; 192 encoded_image->last_referenced_frame_id = latest_frame_id_to_reference; 193 encoded_image->frame_id = ++last_encoded_frame_id_; 194 195 VLOG(1) << "VP8 encoded frame:" << static_cast<int>(encoded_image->frame_id) 196 << " sized:" << total_size; 197 198 if (encoded_image->key_frame) { 199 key_frame_requested_ = false; 200 201 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { 202 used_buffers_frame_id_[i] = encoded_image->frame_id; 203 } 204 // We can pick any buffer as last_used_vp8_buffer_ since we update 205 // them all. 206 last_used_vp8_buffer_ = buffer_to_update; 207 } else { 208 if (buffer_to_update != kNoBuffer) { 209 acked_frame_buffers_[buffer_to_update] = false; 210 used_buffers_frame_id_[buffer_to_update] = encoded_image->frame_id; 211 last_used_vp8_buffer_ = buffer_to_update; 212 } 213 } 214 return true; 215 } 216 217 void Vp8Encoder::GetCodecReferenceFlags(vpx_codec_flags_t* flags) { 218 if (!use_multiple_video_buffers_) return; 219 220 // We need to reference something. 221 DCHECK(acked_frame_buffers_[kAltRefBuffer] || 222 acked_frame_buffers_[kGoldenBuffer] || 223 acked_frame_buffers_[kLastBuffer]) << "Invalid state"; 224 225 if (!acked_frame_buffers_[kAltRefBuffer]) { 226 *flags |= VP8_EFLAG_NO_REF_ARF; 227 } 228 if (!acked_frame_buffers_[kGoldenBuffer]) { 229 *flags |= VP8_EFLAG_NO_REF_GF; 230 } 231 if (!acked_frame_buffers_[kLastBuffer]) { 232 *flags |= VP8_EFLAG_NO_REF_LAST; 233 } 234 } 235 236 uint32 Vp8Encoder::GetLatestFrameIdToReference() { 237 if (!use_multiple_video_buffers_) return last_encoded_frame_id_; 238 239 int latest_frame_id_to_reference = -1; 240 if (acked_frame_buffers_[kAltRefBuffer]) { 241 latest_frame_id_to_reference = used_buffers_frame_id_[kAltRefBuffer]; 242 } 243 if (acked_frame_buffers_[kGoldenBuffer]) { 244 if (latest_frame_id_to_reference == -1) { 245 latest_frame_id_to_reference = used_buffers_frame_id_[kGoldenBuffer]; 246 } else { 247 if (IsNewerFrameId(used_buffers_frame_id_[kGoldenBuffer], 248 latest_frame_id_to_reference)) { 249 latest_frame_id_to_reference = used_buffers_frame_id_[kGoldenBuffer]; 250 } 251 } 252 } 253 if (acked_frame_buffers_[kLastBuffer]) { 254 if (latest_frame_id_to_reference == -1) { 255 latest_frame_id_to_reference = used_buffers_frame_id_[kLastBuffer]; 256 } else { 257 if (IsNewerFrameId(used_buffers_frame_id_[kLastBuffer], 258 latest_frame_id_to_reference)) { 259 latest_frame_id_to_reference = used_buffers_frame_id_[kLastBuffer]; 260 } 261 } 262 } 263 DCHECK(latest_frame_id_to_reference != -1) << "Invalid state"; 264 return static_cast<uint32>(latest_frame_id_to_reference); 265 } 266 267 Vp8Encoder::Vp8Buffers Vp8Encoder::GetNextBufferToUpdate() { 268 // Update at most one buffer, except for key-frames. 269 270 Vp8Buffers buffer_to_update; 271 if (number_of_repeated_buffers_ < max_number_of_repeated_buffers_in_a_row_) { 272 // TODO(pwestin): experiment with this. The issue with only this change is 273 // that we can end up with only 4 frames in flight when we expect 6. 274 // buffer_to_update = last_used_vp8_buffer_; 275 buffer_to_update = kNoBuffer; 276 ++number_of_repeated_buffers_; 277 } else { 278 number_of_repeated_buffers_ = 0; 279 switch (last_used_vp8_buffer_) { 280 case kAltRefBuffer: 281 buffer_to_update = kLastBuffer; 282 VLOG(1) << "VP8 update last buffer"; 283 break; 284 case kLastBuffer: 285 buffer_to_update = kGoldenBuffer; 286 VLOG(1) << "VP8 update golden buffer"; 287 break; 288 case kGoldenBuffer: 289 buffer_to_update = kAltRefBuffer; 290 VLOG(1) << "VP8 update alt-ref buffer"; 291 break; 292 case kNoBuffer: 293 DCHECK(false) << "Invalid state"; 294 break; 295 } 296 } 297 return buffer_to_update; 298 } 299 300 void Vp8Encoder::GetCodecUpdateFlags(Vp8Buffers buffer_to_update, 301 vpx_codec_flags_t* flags) { 302 if (!use_multiple_video_buffers_) return; 303 304 // Update at most one buffer, except for key-frames. 305 switch (buffer_to_update) { 306 case kAltRefBuffer: 307 *flags |= VP8_EFLAG_NO_UPD_GF; 308 *flags |= VP8_EFLAG_NO_UPD_LAST; 309 break; 310 case kLastBuffer: 311 *flags |= VP8_EFLAG_NO_UPD_GF; 312 *flags |= VP8_EFLAG_NO_UPD_ARF; 313 break; 314 case kGoldenBuffer: 315 *flags |= VP8_EFLAG_NO_UPD_ARF; 316 *flags |= VP8_EFLAG_NO_UPD_LAST; 317 break; 318 case kNoBuffer: 319 *flags |= VP8_EFLAG_NO_UPD_ARF; 320 *flags |= VP8_EFLAG_NO_UPD_GF; 321 *flags |= VP8_EFLAG_NO_UPD_LAST; 322 *flags |= VP8_EFLAG_NO_UPD_ENTROPY; 323 break; 324 } 325 } 326 327 void Vp8Encoder::UpdateRates(uint32 new_bitrate) { 328 uint32 new_bitrate_kbit = new_bitrate / 1000; 329 if (config_->rc_target_bitrate == new_bitrate_kbit) return; 330 331 config_->rc_target_bitrate = new_bitrate_kbit; 332 333 // Update encoder context. 334 if (vpx_codec_enc_config_set(encoder_, config_.get())) { 335 DCHECK(false) << "Invalid return value"; 336 } 337 } 338 339 void Vp8Encoder::LatestFrameIdToReference(uint32 frame_id) { 340 if (!use_multiple_video_buffers_) return; 341 342 VLOG(1) << "VP8 ok to reference frame:" << static_cast<int>(frame_id); 343 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { 344 if (frame_id == used_buffers_frame_id_[i]) { 345 acked_frame_buffers_[i] = true; 346 } 347 } 348 } 349 350 void Vp8Encoder::GenerateKeyFrame() { 351 key_frame_requested_ = true; 352 } 353 354 // Calculate the max size of the key frame relative to a normal delta frame. 355 uint32 Vp8Encoder::MaxIntraTarget(uint32 optimal_buffer_size_ms) const { 356 // Set max to the optimal buffer level (normalized by target BR), 357 // and scaled by a scale_parameter. 358 // Max target size = scalePar * optimalBufferSize * targetBR[Kbps]. 359 // This values is presented in percentage of perFrameBw: 360 // perFrameBw = targetBR[Kbps] * 1000 / frameRate. 361 // The target in % is as follows: 362 363 float scale_parameter = 0.5; 364 uint32 target_pct = optimal_buffer_size_ms * scale_parameter * 365 cast_config_.max_frame_rate / 10; 366 367 // Don't go below 3 times the per frame bandwidth. 368 return std::max(target_pct, kMinIntra); 369 } 370 371 } // namespace cast 372 } // namespace media 373
