1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "remoting/codec/video_encoder_vpx.h" 6 7 #include "base/bind.h" 8 #include "base/command_line.h" 9 #include "base/logging.h" 10 #include "base/sys_info.h" 11 #include "remoting/base/util.h" 12 #include "remoting/proto/video.pb.h" 13 #include "third_party/libyuv/include/libyuv/convert_from_argb.h" 14 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h" 15 #include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h" 16 #include "third_party/webrtc/modules/desktop_capture/desktop_region.h" 17 18 extern "C" { 19 #define VPX_CODEC_DISABLE_COMPAT 1 20 #include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h" 21 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h" 22 } 23 24 namespace remoting { 25 26 namespace { 27 28 // Name of command-line flag to enable VP9 to use I444 by default. 29 const char kEnableI444SwitchName[] = "enable-i444"; 30 31 // Number of bytes in an RGBx pixel. 32 const int kBytesPerRgbPixel = 4; 33 34 // Defines the dimension of a macro block. This is used to compute the active 35 // map for the encoder. 36 const int kMacroBlockSize = 16; 37 38 // Magic encoder profile numbers for I420 and I444 input formats. 39 const int kVp9I420ProfileNumber = 0; 40 const int kVp9I444ProfileNumber = 1; 41 42 void SetCommonCodecParameters(const webrtc::DesktopSize& size, 43 vpx_codec_enc_cfg_t* config) { 44 // Use millisecond granularity time base. 45 config->g_timebase.num = 1; 46 config->g_timebase.den = 1000; 47 48 // Adjust default target bit-rate to account for actual desktop size. 49 config->rc_target_bitrate = size.width() * size.height() * 50 config->rc_target_bitrate / config->g_w / config->g_h; 51 52 config->g_w = size.width(); 53 config->g_h = size.height(); 54 config->g_pass = VPX_RC_ONE_PASS; 55 56 // Start emitting packets immediately. 57 config->g_lag_in_frames = 0; 58 59 // Using 2 threads gives a great boost in performance for most systems with 60 // adequate processing power. NB: Going to multiple threads on low end 61 // windows systems can really hurt performance. 62 // http://crbug.com/99179 63 config->g_threads = (base::SysInfo::NumberOfProcessors() > 2) ? 2 : 1; 64 } 65 66 ScopedVpxCodec CreateVP8Codec(const webrtc::DesktopSize& size) { 67 ScopedVpxCodec codec(new vpx_codec_ctx_t); 68 69 // Configure the encoder. 70 vpx_codec_enc_cfg_t config; 71 const vpx_codec_iface_t* algo = vpx_codec_vp8_cx(); 72 CHECK(algo); 73 vpx_codec_err_t ret = vpx_codec_enc_config_default(algo, &config, 0); 74 if (ret != VPX_CODEC_OK) 75 return ScopedVpxCodec(); 76 77 SetCommonCodecParameters(size, &config); 78 79 // Value of 2 means using the real time profile. This is basically a 80 // redundant option since we explicitly select real time mode when doing 81 // encoding. 82 config.g_profile = 2; 83 84 // Clamping the quantizer constrains the worst-case quality and CPU usage. 85 config.rc_min_quantizer = 20; 86 config.rc_max_quantizer = 30; 87 88 if (vpx_codec_enc_init(codec.get(), algo, &config, 0)) 89 return ScopedVpxCodec(); 90 91 // Value of 16 will have the smallest CPU load. This turns off subpixel 92 // motion search. 93 if (vpx_codec_control(codec.get(), VP8E_SET_CPUUSED, 16)) 94 return ScopedVpxCodec(); 95 96 // Use the lowest level of noise sensitivity so as to spend less time 97 // on motion estimation and inter-prediction mode. 98 if (vpx_codec_control(codec.get(), VP8E_SET_NOISE_SENSITIVITY, 0)) 99 return ScopedVpxCodec(); 100 101 return codec.Pass(); 102 } 103 104 ScopedVpxCodec CreateVP9Codec(const webrtc::DesktopSize& size, 105 bool lossless_color, 106 bool lossless_encode) { 107 ScopedVpxCodec codec(new vpx_codec_ctx_t); 108 109 // Configure the encoder. 110 vpx_codec_enc_cfg_t config; 111 const vpx_codec_iface_t* algo = vpx_codec_vp9_cx(); 112 CHECK(algo); 113 vpx_codec_err_t ret = vpx_codec_enc_config_default(algo, &config, 0); 114 if (ret != VPX_CODEC_OK) 115 return ScopedVpxCodec(); 116 117 SetCommonCodecParameters(size, &config); 118 119 // Configure VP9 for I420 or I444 source frames. 120 config.g_profile = 121 lossless_color ? kVp9I444ProfileNumber : kVp9I420ProfileNumber; 122 123 if (lossless_encode) { 124 // Disable quantization entirely, putting the encoder in "lossless" mode. 125 config.rc_min_quantizer = 0; 126 config.rc_max_quantizer = 0; 127 } else { 128 // Lossy encode using the same settings as for VP8. 129 config.rc_min_quantizer = 20; 130 config.rc_max_quantizer = 30; 131 } 132 133 if (vpx_codec_enc_init(codec.get(), algo, &config, 0)) 134 return ScopedVpxCodec(); 135 136 // Request the lowest-CPU usage that VP9 supports, which depends on whether 137 // we are encoding lossy or lossless. 138 // Note that this is configured via the same parameter as for VP8. 139 int cpu_used = lossless_encode ? 5 : 7; 140 if (vpx_codec_control(codec.get(), VP8E_SET_CPUUSED, cpu_used)) 141 return ScopedVpxCodec(); 142 143 // Use the lowest level of noise sensitivity so as to spend less time 144 // on motion estimation and inter-prediction mode. 145 if (vpx_codec_control(codec.get(), VP9E_SET_NOISE_SENSITIVITY, 0)) 146 return ScopedVpxCodec(); 147 148 return codec.Pass(); 149 } 150 151 void CreateImage(bool use_i444, 152 const webrtc::DesktopSize& size, 153 scoped_ptr<vpx_image_t>* out_image, 154 scoped_ptr<uint8[]>* out_image_buffer) { 155 DCHECK(!size.is_empty()); 156 157 scoped_ptr<vpx_image_t> image(new vpx_image_t()); 158 memset(image.get(), 0, sizeof(vpx_image_t)); 159 160 // libvpx seems to require both to be assigned. 161 image->d_w = size.width(); 162 image->w = size.width(); 163 image->d_h = size.height(); 164 image->h = size.height(); 165 166 // libvpx should derive chroma shifts from|fmt| but currently has a bug: 167 // https://code.google.com/p/webm/issues/detail?id=627 168 if (use_i444) { 169 image->fmt = VPX_IMG_FMT_I444; 170 image->x_chroma_shift = 0; 171 image->y_chroma_shift = 0; 172 } else { // I420 173 image->fmt = VPX_IMG_FMT_YV12; 174 image->x_chroma_shift = 1; 175 image->y_chroma_shift = 1; 176 } 177 178 // libyuv's fast-path requires 16-byte aligned pointers and strides, so pad 179 // the Y, U and V planes' strides to multiples of 16 bytes. 180 const int y_stride = ((image->w - 1) & ~15) + 16; 181 const int uv_unaligned_stride = y_stride >> image->x_chroma_shift; 182 const int uv_stride = ((uv_unaligned_stride - 1) & ~15) + 16; 183 184 // libvpx accesses the source image in macro blocks, and will over-read 185 // if the image is not padded out to the next macroblock: crbug.com/119633. 186 // Pad the Y, U and V planes' height out to compensate. 187 // Assuming macroblocks are 16x16, aligning the planes' strides above also 188 // macroblock aligned them. 189 DCHECK_EQ(16, kMacroBlockSize); 190 const int y_rows = ((image->h - 1) & ~(kMacroBlockSize-1)) + kMacroBlockSize; 191 const int uv_rows = y_rows >> image->y_chroma_shift; 192 193 // Allocate a YUV buffer large enough for the aligned data & padding. 194 const int buffer_size = y_stride * y_rows + 2*uv_stride * uv_rows; 195 scoped_ptr<uint8[]> image_buffer(new uint8[buffer_size]); 196 197 // Reset image value to 128 so we just need to fill in the y plane. 198 memset(image_buffer.get(), 128, buffer_size); 199 200 // Fill in the information for |image_|. 201 unsigned char* uchar_buffer = 202 reinterpret_cast<unsigned char*>(image_buffer.get()); 203 image->planes[0] = uchar_buffer; 204 image->planes[1] = image->planes[0] + y_stride * y_rows; 205 image->planes[2] = image->planes[1] + uv_stride * uv_rows; 206 image->stride[0] = y_stride; 207 image->stride[1] = uv_stride; 208 image->stride[2] = uv_stride; 209 210 *out_image = image.Pass(); 211 *out_image_buffer = image_buffer.Pass(); 212 } 213 214 } // namespace 215 216 // static 217 scoped_ptr<VideoEncoderVpx> VideoEncoderVpx::CreateForVP8() { 218 return scoped_ptr<VideoEncoderVpx>(new VideoEncoderVpx(false)); 219 } 220 221 // static 222 scoped_ptr<VideoEncoderVpx> VideoEncoderVpx::CreateForVP9() { 223 return scoped_ptr<VideoEncoderVpx>(new VideoEncoderVpx(true)); 224 } 225 226 VideoEncoderVpx::~VideoEncoderVpx() {} 227 228 void VideoEncoderVpx::SetLosslessEncode(bool want_lossless) { 229 if (use_vp9_ && (want_lossless != lossless_encode_)) { 230 lossless_encode_ = want_lossless; 231 codec_.reset(); // Force encoder re-initialization. 232 } 233 } 234 235 void VideoEncoderVpx::SetLosslessColor(bool want_lossless) { 236 if (use_vp9_ && (want_lossless != lossless_color_)) { 237 lossless_color_ = want_lossless; 238 codec_.reset(); // Force encoder re-initialization. 239 } 240 } 241 242 scoped_ptr<VideoPacket> VideoEncoderVpx::Encode( 243 const webrtc::DesktopFrame& frame) { 244 DCHECK_LE(32, frame.size().width()); 245 DCHECK_LE(32, frame.size().height()); 246 247 base::TimeTicks encode_start_time = base::TimeTicks::Now(); 248 249 if (!codec_ || 250 !frame.size().equals(webrtc::DesktopSize(image_->w, image_->h))) { 251 bool ret = Initialize(frame.size()); 252 // TODO(hclam): Handle error better. 253 CHECK(ret) << "Initialization of encoder failed"; 254 255 // Set now as the base for timestamp calculation. 256 timestamp_base_ = encode_start_time; 257 } 258 259 // Convert the updated capture data ready for encode. 260 webrtc::DesktopRegion updated_region; 261 PrepareImage(frame, &updated_region); 262 263 // Update active map based on updated region. 264 PrepareActiveMap(updated_region); 265 266 // Apply active map to the encoder. 267 vpx_active_map_t act_map; 268 act_map.rows = active_map_height_; 269 act_map.cols = active_map_width_; 270 act_map.active_map = active_map_.get(); 271 if (vpx_codec_control(codec_.get(), VP8E_SET_ACTIVEMAP, &act_map)) { 272 LOG(ERROR) << "Unable to apply active map"; 273 } 274 275 // Do the actual encoding. 276 int timestamp = (encode_start_time - timestamp_base_).InMilliseconds(); 277 vpx_codec_err_t ret = vpx_codec_encode( 278 codec_.get(), image_.get(), timestamp, 1, 0, VPX_DL_REALTIME); 279 DCHECK_EQ(ret, VPX_CODEC_OK) 280 << "Encoding error: " << vpx_codec_err_to_string(ret) << "\n" 281 << "Details: " << vpx_codec_error(codec_.get()) << "\n" 282 << vpx_codec_error_detail(codec_.get()); 283 284 // Read the encoded data. 285 vpx_codec_iter_t iter = NULL; 286 bool got_data = false; 287 288 // TODO(hclam): Make sure we get exactly one frame from the packet. 289 // TODO(hclam): We should provide the output buffer to avoid one copy. 290 scoped_ptr<VideoPacket> packet( 291 helper_.CreateVideoPacketWithUpdatedRegion(frame, updated_region)); 292 packet->mutable_format()->set_encoding(VideoPacketFormat::ENCODING_VP8); 293 294 while (!got_data) { 295 const vpx_codec_cx_pkt_t* vpx_packet = 296 vpx_codec_get_cx_data(codec_.get(), &iter); 297 if (!vpx_packet) 298 continue; 299 300 switch (vpx_packet->kind) { 301 case VPX_CODEC_CX_FRAME_PKT: 302 got_data = true; 303 packet->set_data(vpx_packet->data.frame.buf, vpx_packet->data.frame.sz); 304 break; 305 default: 306 break; 307 } 308 } 309 310 // Note the time taken to encode the pixel data. 311 packet->set_encode_time_ms( 312 (base::TimeTicks::Now() - encode_start_time).InMillisecondsRoundedUp()); 313 314 return packet.Pass(); 315 } 316 317 VideoEncoderVpx::VideoEncoderVpx(bool use_vp9) 318 : use_vp9_(use_vp9), 319 lossless_encode_(false), 320 lossless_color_(false), 321 active_map_width_(0), 322 active_map_height_(0) { 323 if (use_vp9_) { 324 // Use I444 colour space, by default, if specified on the command-line. 325 if (CommandLine::ForCurrentProcess()->HasSwitch(kEnableI444SwitchName)) { 326 SetLosslessColor(true); 327 } 328 } 329 } 330 331 bool VideoEncoderVpx::Initialize(const webrtc::DesktopSize& size) { 332 DCHECK(use_vp9_ || !lossless_color_); 333 DCHECK(use_vp9_ || !lossless_encode_); 334 335 codec_.reset(); 336 337 // (Re)Create the VPX image structure and pixel buffer. 338 CreateImage(lossless_color_, size, &image_, &image_buffer_); 339 340 // Initialize active map. 341 active_map_width_ = (image_->w + kMacroBlockSize - 1) / kMacroBlockSize; 342 active_map_height_ = (image_->h + kMacroBlockSize - 1) / kMacroBlockSize; 343 active_map_.reset(new uint8[active_map_width_ * active_map_height_]); 344 345 // (Re)Initialize the codec. 346 if (use_vp9_) { 347 codec_ = CreateVP9Codec(size, lossless_color_, lossless_encode_); 348 } else { 349 codec_ = CreateVP8Codec(size); 350 } 351 352 return codec_; 353 } 354 355 void VideoEncoderVpx::PrepareImage(const webrtc::DesktopFrame& frame, 356 webrtc::DesktopRegion* updated_region) { 357 if (frame.updated_region().is_empty()) { 358 updated_region->Clear(); 359 return; 360 } 361 362 // Align the region to macroblocks, to avoid encoding artefacts. 363 // This also ensures that all rectangles have even-aligned top-left, which 364 // is required for ConvertRGBToYUVWithRect() to work. 365 std::vector<webrtc::DesktopRect> aligned_rects; 366 for (webrtc::DesktopRegion::Iterator r(frame.updated_region()); 367 !r.IsAtEnd(); r.Advance()) { 368 const webrtc::DesktopRect& rect = r.rect(); 369 aligned_rects.push_back(AlignRect(webrtc::DesktopRect::MakeLTRB( 370 rect.left(), rect.top(), rect.right(), rect.bottom()))); 371 } 372 DCHECK(!aligned_rects.empty()); 373 updated_region->Clear(); 374 updated_region->AddRects(&aligned_rects[0], aligned_rects.size()); 375 376 // Clip back to the screen dimensions, in case they're not macroblock aligned. 377 // The conversion routines don't require even width & height, so this is safe 378 // even if the source dimensions are not even. 379 updated_region->IntersectWith( 380 webrtc::DesktopRect::MakeWH(image_->w, image_->h)); 381 382 // Convert the updated region to YUV ready for encoding. 383 const uint8* rgb_data = frame.data(); 384 const int rgb_stride = frame.stride(); 385 const int y_stride = image_->stride[0]; 386 DCHECK_EQ(image_->stride[1], image_->stride[2]); 387 const int uv_stride = image_->stride[1]; 388 uint8* y_data = image_->planes[0]; 389 uint8* u_data = image_->planes[1]; 390 uint8* v_data = image_->planes[2]; 391 392 switch (image_->fmt) { 393 case VPX_IMG_FMT_I444: 394 for (webrtc::DesktopRegion::Iterator r(*updated_region); !r.IsAtEnd(); 395 r.Advance()) { 396 const webrtc::DesktopRect& rect = r.rect(); 397 int rgb_offset = rgb_stride * rect.top() + 398 rect.left() * kBytesPerRgbPixel; 399 int yuv_offset = uv_stride * rect.top() + rect.left(); 400 libyuv::ARGBToI444(rgb_data + rgb_offset, rgb_stride, 401 y_data + yuv_offset, y_stride, 402 u_data + yuv_offset, uv_stride, 403 v_data + yuv_offset, uv_stride, 404 rect.width(), rect.height()); 405 } 406 break; 407 case VPX_IMG_FMT_YV12: 408 for (webrtc::DesktopRegion::Iterator r(*updated_region); !r.IsAtEnd(); 409 r.Advance()) { 410 const webrtc::DesktopRect& rect = r.rect(); 411 int rgb_offset = rgb_stride * rect.top() + 412 rect.left() * kBytesPerRgbPixel; 413 int y_offset = y_stride * rect.top() + rect.left(); 414 int uv_offset = uv_stride * rect.top() / 2 + rect.left() / 2; 415 libyuv::ARGBToI420(rgb_data + rgb_offset, rgb_stride, 416 y_data + y_offset, y_stride, 417 u_data + uv_offset, uv_stride, 418 v_data + uv_offset, uv_stride, 419 rect.width(), rect.height()); 420 } 421 break; 422 default: 423 NOTREACHED(); 424 break; 425 } 426 } 427 428 void VideoEncoderVpx::PrepareActiveMap( 429 const webrtc::DesktopRegion& updated_region) { 430 // Clear active map first. 431 memset(active_map_.get(), 0, active_map_width_ * active_map_height_); 432 433 // Mark updated areas active. 434 for (webrtc::DesktopRegion::Iterator r(updated_region); !r.IsAtEnd(); 435 r.Advance()) { 436 const webrtc::DesktopRect& rect = r.rect(); 437 int left = rect.left() / kMacroBlockSize; 438 int right = (rect.right() - 1) / kMacroBlockSize; 439 int top = rect.top() / kMacroBlockSize; 440 int bottom = (rect.bottom() - 1) / kMacroBlockSize; 441 DCHECK_LT(right, active_map_width_); 442 DCHECK_LT(bottom, active_map_height_); 443 444 uint8* map = active_map_.get() + top * active_map_width_; 445 for (int y = top; y <= bottom; ++y) { 446 for (int x = left; x <= right; ++x) 447 map[x] = 1; 448 map += active_map_width_; 449 } 450 } 451 } 452 453 } // namespace remoting 454