1 // Copyright 2014 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "content/renderer/pepper/video_decoder_shim.h" 6 7 #include <GLES2/gl2.h> 8 #include <GLES2/gl2ext.h> 9 #include <GLES2/gl2extchromium.h> 10 11 #include "base/bind.h" 12 #include "base/numerics/safe_conversions.h" 13 #include "base/single_thread_task_runner.h" 14 #include "content/public/renderer/render_thread.h" 15 #include "content/renderer/pepper/pepper_video_decoder_host.h" 16 #include "content/renderer/render_thread_impl.h" 17 #include "gpu/command_buffer/client/gles2_implementation.h" 18 #include "media/base/decoder_buffer.h" 19 #include "media/base/limits.h" 20 #include "media/base/video_decoder.h" 21 #include "media/filters/ffmpeg_video_decoder.h" 22 #include "media/filters/vpx_video_decoder.h" 23 #include "media/video/picture.h" 24 #include "media/video/video_decode_accelerator.h" 25 #include "ppapi/c/pp_errors.h" 26 #include "third_party/libyuv/include/libyuv.h" 27 #include "webkit/common/gpu/context_provider_web_context.h" 28 29 namespace content { 30 31 struct VideoDecoderShim::PendingDecode { 32 PendingDecode(uint32_t decode_id, 33 const scoped_refptr<media::DecoderBuffer>& buffer); 34 ~PendingDecode(); 35 36 const uint32_t decode_id; 37 const scoped_refptr<media::DecoderBuffer> buffer; 38 }; 39 40 VideoDecoderShim::PendingDecode::PendingDecode( 41 uint32_t decode_id, 42 const scoped_refptr<media::DecoderBuffer>& buffer) 43 : decode_id(decode_id), buffer(buffer) { 44 } 45 46 VideoDecoderShim::PendingDecode::~PendingDecode() { 47 } 48 49 struct VideoDecoderShim::PendingFrame { 50 explicit PendingFrame(uint32_t decode_id); 51 PendingFrame(uint32_t decode_id, 52 const gfx::Size& coded_size, 53 const gfx::Rect& visible_rect); 54 ~PendingFrame(); 55 56 const uint32_t decode_id; 57 const gfx::Size coded_size; 58 const gfx::Rect visible_rect; 59 std::vector<uint8_t> argb_pixels; 60 61 private: 62 // This could be expensive to copy, so guard against that. 63 DISALLOW_COPY_AND_ASSIGN(PendingFrame); 64 }; 65 66 VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id) 67 : decode_id(decode_id) { 68 } 69 70 VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id, 71 const gfx::Size& coded_size, 72 const gfx::Rect& visible_rect) 73 : decode_id(decode_id), 74 coded_size(coded_size), 75 visible_rect(visible_rect), 76 argb_pixels(coded_size.width() * coded_size.height() * 4) { 77 } 78 79 VideoDecoderShim::PendingFrame::~PendingFrame() { 80 } 81 82 // DecoderImpl runs the underlying VideoDecoder on the media thread, receiving 83 // calls from the VideoDecodeShim on the main thread and sending results back. 84 // This class is constructed on the main thread, but used and destructed on the 85 // media thread. 86 class VideoDecoderShim::DecoderImpl { 87 public: 88 explicit DecoderImpl(const base::WeakPtr<VideoDecoderShim>& proxy); 89 ~DecoderImpl(); 90 91 void Initialize(media::VideoDecoderConfig config); 92 void Decode(uint32_t decode_id, scoped_refptr<media::DecoderBuffer> buffer); 93 void Reset(); 94 void Stop(); 95 96 private: 97 void OnPipelineStatus(media::PipelineStatus status); 98 void DoDecode(); 99 void OnDecodeComplete(uint32_t decode_id, media::VideoDecoder::Status status); 100 void OnOutputComplete(const scoped_refptr<media::VideoFrame>& frame); 101 void OnResetComplete(); 102 103 // WeakPtr is bound to main_message_loop_. Use only in shim callbacks. 104 base::WeakPtr<VideoDecoderShim> shim_; 105 scoped_ptr<media::VideoDecoder> decoder_; 106 scoped_refptr<base::MessageLoopProxy> main_message_loop_; 107 // Queue of decodes waiting for the decoder. 108 typedef std::queue<PendingDecode> PendingDecodeQueue; 109 PendingDecodeQueue pending_decodes_; 110 int max_decodes_at_decoder_; 111 int num_decodes_at_decoder_; 112 // VideoDecoder returns pictures without information about the decode buffer 113 // that generated it. Save the decode_id from the last decode that completed, 114 // which is close for most decoders, which only decode one buffer at a time. 115 uint32_t decode_id_; 116 }; 117 118 VideoDecoderShim::DecoderImpl::DecoderImpl( 119 const base::WeakPtr<VideoDecoderShim>& proxy) 120 : shim_(proxy), 121 main_message_loop_(base::MessageLoopProxy::current()), 122 max_decodes_at_decoder_(0), 123 num_decodes_at_decoder_(0), 124 decode_id_(0) { 125 } 126 127 VideoDecoderShim::DecoderImpl::~DecoderImpl() { 128 DCHECK(pending_decodes_.empty()); 129 } 130 131 void VideoDecoderShim::DecoderImpl::Initialize( 132 media::VideoDecoderConfig config) { 133 DCHECK(!decoder_); 134 #if !defined(MEDIA_DISABLE_LIBVPX) 135 if (config.codec() == media::kCodecVP9) { 136 decoder_.reset( 137 new media::VpxVideoDecoder(base::MessageLoopProxy::current())); 138 } else 139 #endif 140 { 141 scoped_ptr<media::FFmpegVideoDecoder> ffmpeg_video_decoder( 142 new media::FFmpegVideoDecoder(base::MessageLoopProxy::current())); 143 ffmpeg_video_decoder->set_decode_nalus(true); 144 decoder_ = ffmpeg_video_decoder.Pass(); 145 } 146 max_decodes_at_decoder_ = decoder_->GetMaxDecodeRequests(); 147 // We can use base::Unretained() safely in decoder callbacks because 148 // |decoder_| is owned by DecoderImpl. During Stop(), the |decoder_| will be 149 // destroyed and all outstanding callbacks will be fired. 150 decoder_->Initialize( 151 config, 152 true /* low_delay */, 153 base::Bind(&VideoDecoderShim::DecoderImpl::OnPipelineStatus, 154 base::Unretained(this)), 155 base::Bind(&VideoDecoderShim::DecoderImpl::OnOutputComplete, 156 base::Unretained(this))); 157 } 158 159 void VideoDecoderShim::DecoderImpl::Decode( 160 uint32_t decode_id, 161 scoped_refptr<media::DecoderBuffer> buffer) { 162 DCHECK(decoder_); 163 pending_decodes_.push(PendingDecode(decode_id, buffer)); 164 DoDecode(); 165 } 166 167 void VideoDecoderShim::DecoderImpl::Reset() { 168 DCHECK(decoder_); 169 // Abort all pending decodes. 170 while (!pending_decodes_.empty()) { 171 const PendingDecode& decode = pending_decodes_.front(); 172 scoped_ptr<PendingFrame> pending_frame(new PendingFrame(decode.decode_id)); 173 main_message_loop_->PostTask(FROM_HERE, 174 base::Bind(&VideoDecoderShim::OnDecodeComplete, 175 shim_, 176 media::VideoDecoder::kAborted, 177 decode.decode_id)); 178 pending_decodes_.pop(); 179 } 180 decoder_->Reset(base::Bind(&VideoDecoderShim::DecoderImpl::OnResetComplete, 181 base::Unretained(this))); 182 } 183 184 void VideoDecoderShim::DecoderImpl::Stop() { 185 DCHECK(decoder_); 186 // Clear pending decodes now. We don't want OnDecodeComplete to call DoDecode 187 // again. 188 while (!pending_decodes_.empty()) 189 pending_decodes_.pop(); 190 decoder_.reset(); 191 // This instance is deleted once we exit this scope. 192 } 193 194 void VideoDecoderShim::DecoderImpl::OnPipelineStatus( 195 media::PipelineStatus status) { 196 int32_t result; 197 switch (status) { 198 case media::PIPELINE_OK: 199 result = PP_OK; 200 break; 201 case media::DECODER_ERROR_NOT_SUPPORTED: 202 result = PP_ERROR_NOTSUPPORTED; 203 break; 204 default: 205 result = PP_ERROR_FAILED; 206 break; 207 } 208 209 // Calculate how many textures the shim should create. 210 uint32_t shim_texture_pool_size = 211 max_decodes_at_decoder_ + media::limits::kMaxVideoFrames; 212 main_message_loop_->PostTask( 213 FROM_HERE, 214 base::Bind(&VideoDecoderShim::OnInitializeComplete, 215 shim_, 216 result, 217 shim_texture_pool_size)); 218 } 219 220 void VideoDecoderShim::DecoderImpl::DoDecode() { 221 while (!pending_decodes_.empty() && 222 num_decodes_at_decoder_ < max_decodes_at_decoder_) { 223 num_decodes_at_decoder_++; 224 const PendingDecode& decode = pending_decodes_.front(); 225 decoder_->Decode( 226 decode.buffer, 227 base::Bind(&VideoDecoderShim::DecoderImpl::OnDecodeComplete, 228 base::Unretained(this), 229 decode.decode_id)); 230 pending_decodes_.pop(); 231 } 232 } 233 234 void VideoDecoderShim::DecoderImpl::OnDecodeComplete( 235 uint32_t decode_id, 236 media::VideoDecoder::Status status) { 237 num_decodes_at_decoder_--; 238 decode_id_ = decode_id; 239 240 int32_t result; 241 switch (status) { 242 case media::VideoDecoder::kOk: 243 case media::VideoDecoder::kAborted: 244 result = PP_OK; 245 break; 246 case media::VideoDecoder::kDecodeError: 247 result = PP_ERROR_RESOURCE_FAILED; 248 break; 249 default: 250 NOTREACHED(); 251 result = PP_ERROR_FAILED; 252 break; 253 } 254 255 main_message_loop_->PostTask( 256 FROM_HERE, 257 base::Bind( 258 &VideoDecoderShim::OnDecodeComplete, shim_, result, decode_id)); 259 260 DoDecode(); 261 } 262 263 void VideoDecoderShim::DecoderImpl::OnOutputComplete( 264 const scoped_refptr<media::VideoFrame>& frame) { 265 scoped_ptr<PendingFrame> pending_frame; 266 if (!frame->end_of_stream()) { 267 pending_frame.reset(new PendingFrame( 268 decode_id_, frame->coded_size(), frame->visible_rect())); 269 // Convert the VideoFrame pixels to ABGR to match VideoDecodeAccelerator. 270 libyuv::I420ToABGR(frame->data(media::VideoFrame::kYPlane), 271 frame->stride(media::VideoFrame::kYPlane), 272 frame->data(media::VideoFrame::kUPlane), 273 frame->stride(media::VideoFrame::kUPlane), 274 frame->data(media::VideoFrame::kVPlane), 275 frame->stride(media::VideoFrame::kVPlane), 276 &pending_frame->argb_pixels.front(), 277 frame->coded_size().width() * 4, 278 frame->coded_size().width(), 279 frame->coded_size().height()); 280 } else { 281 pending_frame.reset(new PendingFrame(decode_id_)); 282 } 283 284 main_message_loop_->PostTask(FROM_HERE, 285 base::Bind(&VideoDecoderShim::OnOutputComplete, 286 shim_, 287 base::Passed(&pending_frame))); 288 } 289 290 void VideoDecoderShim::DecoderImpl::OnResetComplete() { 291 main_message_loop_->PostTask( 292 FROM_HERE, base::Bind(&VideoDecoderShim::OnResetComplete, shim_)); 293 } 294 295 VideoDecoderShim::VideoDecoderShim(PepperVideoDecoderHost* host) 296 : state_(UNINITIALIZED), 297 host_(host), 298 media_task_runner_( 299 RenderThreadImpl::current()->GetMediaThreadTaskRunner()), 300 context_provider_( 301 RenderThreadImpl::current()->SharedMainThreadContextProvider()), 302 texture_pool_size_(0), 303 num_pending_decodes_(0), 304 weak_ptr_factory_(this) { 305 DCHECK(host_); 306 DCHECK(media_task_runner_.get()); 307 DCHECK(context_provider_.get()); 308 decoder_impl_.reset(new DecoderImpl(weak_ptr_factory_.GetWeakPtr())); 309 } 310 311 VideoDecoderShim::~VideoDecoderShim() { 312 DCHECK(RenderThreadImpl::current()); 313 // Delete any remaining textures. 314 TextureIdMap::iterator it = texture_id_map_.begin(); 315 for (; it != texture_id_map_.end(); ++it) 316 DeleteTexture(it->second); 317 texture_id_map_.clear(); 318 319 FlushCommandBuffer(); 320 321 weak_ptr_factory_.InvalidateWeakPtrs(); 322 // No more callbacks from the delegate will be received now. 323 324 // The callback now holds the only reference to the DecoderImpl, which will be 325 // deleted when Stop completes. 326 media_task_runner_->PostTask( 327 FROM_HERE, 328 base::Bind(&VideoDecoderShim::DecoderImpl::Stop, 329 base::Owned(decoder_impl_.release()))); 330 } 331 332 bool VideoDecoderShim::Initialize( 333 media::VideoCodecProfile profile, 334 media::VideoDecodeAccelerator::Client* client) { 335 DCHECK_EQ(client, host_); 336 DCHECK(RenderThreadImpl::current()); 337 DCHECK_EQ(state_, UNINITIALIZED); 338 media::VideoCodec codec = media::kUnknownVideoCodec; 339 if (profile <= media::H264PROFILE_MAX) 340 codec = media::kCodecH264; 341 else if (profile <= media::VP8PROFILE_MAX) 342 codec = media::kCodecVP8; 343 else if (profile <= media::VP9PROFILE_MAX) 344 codec = media::kCodecVP9; 345 DCHECK_NE(codec, media::kUnknownVideoCodec); 346 347 media::VideoDecoderConfig config( 348 codec, 349 profile, 350 media::VideoFrame::YV12, 351 gfx::Size(32, 24), // Small sizes that won't fail. 352 gfx::Rect(32, 24), 353 gfx::Size(32, 24), 354 NULL /* extra_data */, // TODO(bbudge) Verify this isn't needed. 355 0 /* extra_data_size */, 356 false /* decryption */); 357 358 media_task_runner_->PostTask( 359 FROM_HERE, 360 base::Bind(&VideoDecoderShim::DecoderImpl::Initialize, 361 base::Unretained(decoder_impl_.get()), 362 config)); 363 // Return success, even though we are asynchronous, to mimic 364 // media::VideoDecodeAccelerator. 365 return true; 366 } 367 368 void VideoDecoderShim::Decode(const media::BitstreamBuffer& bitstream_buffer) { 369 DCHECK(RenderThreadImpl::current()); 370 DCHECK_EQ(state_, DECODING); 371 372 // We need the address of the shared memory, so we can copy the buffer. 373 const uint8_t* buffer = host_->DecodeIdToAddress(bitstream_buffer.id()); 374 DCHECK(buffer); 375 376 media_task_runner_->PostTask( 377 FROM_HERE, 378 base::Bind( 379 &VideoDecoderShim::DecoderImpl::Decode, 380 base::Unretained(decoder_impl_.get()), 381 bitstream_buffer.id(), 382 media::DecoderBuffer::CopyFrom(buffer, bitstream_buffer.size()))); 383 num_pending_decodes_++; 384 } 385 386 void VideoDecoderShim::AssignPictureBuffers( 387 const std::vector<media::PictureBuffer>& buffers) { 388 DCHECK(RenderThreadImpl::current()); 389 DCHECK_EQ(state_, DECODING); 390 if (buffers.empty()) { 391 NOTREACHED(); 392 return; 393 } 394 DCHECK_EQ(buffers.size(), pending_texture_mailboxes_.size()); 395 GLuint num_textures = base::checked_cast<GLuint>(buffers.size()); 396 std::vector<uint32_t> local_texture_ids(num_textures); 397 gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL(); 398 for (uint32_t i = 0; i < num_textures; i++) { 399 local_texture_ids[i] = gles2->CreateAndConsumeTextureCHROMIUM( 400 GL_TEXTURE_2D, pending_texture_mailboxes_[i].name); 401 // Map the plugin texture id to the local texture id. 402 uint32_t plugin_texture_id = buffers[i].texture_id(); 403 texture_id_map_[plugin_texture_id] = local_texture_ids[i]; 404 available_textures_.insert(plugin_texture_id); 405 } 406 pending_texture_mailboxes_.clear(); 407 SendPictures(); 408 } 409 410 void VideoDecoderShim::ReusePictureBuffer(int32 picture_buffer_id) { 411 DCHECK(RenderThreadImpl::current()); 412 uint32_t texture_id = static_cast<uint32_t>(picture_buffer_id); 413 if (textures_to_dismiss_.find(texture_id) != textures_to_dismiss_.end()) { 414 DismissTexture(texture_id); 415 } else if (texture_id_map_.find(texture_id) != texture_id_map_.end()) { 416 available_textures_.insert(texture_id); 417 SendPictures(); 418 } else { 419 NOTREACHED(); 420 } 421 } 422 423 void VideoDecoderShim::Flush() { 424 DCHECK(RenderThreadImpl::current()); 425 DCHECK_EQ(state_, DECODING); 426 state_ = FLUSHING; 427 } 428 429 void VideoDecoderShim::Reset() { 430 DCHECK(RenderThreadImpl::current()); 431 DCHECK_EQ(state_, DECODING); 432 state_ = RESETTING; 433 media_task_runner_->PostTask( 434 FROM_HERE, 435 base::Bind(&VideoDecoderShim::DecoderImpl::Reset, 436 base::Unretained(decoder_impl_.get()))); 437 } 438 439 void VideoDecoderShim::Destroy() { 440 // This will be called, but our destructor does the actual work. 441 } 442 443 void VideoDecoderShim::OnInitializeComplete(int32_t result, 444 uint32_t texture_pool_size) { 445 DCHECK(RenderThreadImpl::current()); 446 DCHECK(host_); 447 448 if (result == PP_OK) { 449 state_ = DECODING; 450 texture_pool_size_ = texture_pool_size; 451 } 452 453 host_->OnInitializeComplete(result); 454 } 455 456 void VideoDecoderShim::OnDecodeComplete(int32_t result, uint32_t decode_id) { 457 DCHECK(RenderThreadImpl::current()); 458 DCHECK(host_); 459 460 if (result == PP_ERROR_RESOURCE_FAILED) { 461 host_->NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE); 462 return; 463 } 464 465 num_pending_decodes_--; 466 completed_decodes_.push(decode_id); 467 468 // If frames are being queued because we're out of textures, don't notify 469 // the host that decode has completed. This exerts "back pressure" to keep 470 // the host from sending buffers that will cause pending_frames_ to grow. 471 if (pending_frames_.empty()) 472 NotifyCompletedDecodes(); 473 } 474 475 void VideoDecoderShim::OnOutputComplete(scoped_ptr<PendingFrame> frame) { 476 DCHECK(RenderThreadImpl::current()); 477 DCHECK(host_); 478 479 if (!frame->argb_pixels.empty()) { 480 if (texture_size_ != frame->coded_size) { 481 // If the size has changed, all current textures must be dismissed. Add 482 // all textures to |textures_to_dismiss_| and dismiss any that aren't in 483 // use by the plugin. We will dismiss the rest as they are recycled. 484 for (TextureIdMap::const_iterator it = texture_id_map_.begin(); 485 it != texture_id_map_.end(); 486 ++it) { 487 textures_to_dismiss_.insert(it->second); 488 } 489 for (TextureIdSet::const_iterator it = available_textures_.begin(); 490 it != available_textures_.end(); 491 ++it) { 492 DismissTexture(*it); 493 } 494 available_textures_.clear(); 495 FlushCommandBuffer(); 496 497 DCHECK(pending_texture_mailboxes_.empty()); 498 for (uint32_t i = 0; i < texture_pool_size_; i++) 499 pending_texture_mailboxes_.push_back(gpu::Mailbox::Generate()); 500 501 host_->RequestTextures(texture_pool_size_, 502 frame->coded_size, 503 GL_TEXTURE_2D, 504 pending_texture_mailboxes_); 505 texture_size_ = frame->coded_size; 506 } 507 508 pending_frames_.push(linked_ptr<PendingFrame>(frame.release())); 509 SendPictures(); 510 } 511 } 512 513 void VideoDecoderShim::SendPictures() { 514 DCHECK(RenderThreadImpl::current()); 515 DCHECK(host_); 516 while (!pending_frames_.empty() && !available_textures_.empty()) { 517 const linked_ptr<PendingFrame>& frame = pending_frames_.front(); 518 519 TextureIdSet::iterator it = available_textures_.begin(); 520 uint32_t texture_id = *it; 521 available_textures_.erase(it); 522 523 uint32_t local_texture_id = texture_id_map_[texture_id]; 524 gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL(); 525 gles2->ActiveTexture(GL_TEXTURE0); 526 gles2->BindTexture(GL_TEXTURE_2D, local_texture_id); 527 gles2->TexImage2D(GL_TEXTURE_2D, 528 0, 529 GL_RGBA, 530 texture_size_.width(), 531 texture_size_.height(), 532 0, 533 GL_RGBA, 534 GL_UNSIGNED_BYTE, 535 &frame->argb_pixels.front()); 536 537 host_->PictureReady( 538 media::Picture(texture_id, frame->decode_id, frame->visible_rect)); 539 pending_frames_.pop(); 540 } 541 542 FlushCommandBuffer(); 543 544 if (pending_frames_.empty()) { 545 // If frames aren't backing up, notify the host of any completed decodes so 546 // it can send more buffers. 547 NotifyCompletedDecodes(); 548 549 if (state_ == FLUSHING && !num_pending_decodes_) { 550 state_ = DECODING; 551 host_->NotifyFlushDone(); 552 } 553 } 554 } 555 556 void VideoDecoderShim::OnResetComplete() { 557 DCHECK(RenderThreadImpl::current()); 558 DCHECK(host_); 559 560 while (!pending_frames_.empty()) 561 pending_frames_.pop(); 562 NotifyCompletedDecodes(); 563 564 // Dismiss any old textures now. 565 while (!textures_to_dismiss_.empty()) 566 DismissTexture(*textures_to_dismiss_.begin()); 567 568 state_ = DECODING; 569 host_->NotifyResetDone(); 570 } 571 572 void VideoDecoderShim::NotifyCompletedDecodes() { 573 while (!completed_decodes_.empty()) { 574 host_->NotifyEndOfBitstreamBuffer(completed_decodes_.front()); 575 completed_decodes_.pop(); 576 } 577 } 578 579 void VideoDecoderShim::DismissTexture(uint32_t texture_id) { 580 DCHECK(host_); 581 textures_to_dismiss_.erase(texture_id); 582 DCHECK(texture_id_map_.find(texture_id) != texture_id_map_.end()); 583 DeleteTexture(texture_id_map_[texture_id]); 584 texture_id_map_.erase(texture_id); 585 host_->DismissPictureBuffer(texture_id); 586 } 587 588 void VideoDecoderShim::DeleteTexture(uint32_t texture_id) { 589 gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL(); 590 gles2->DeleteTextures(1, &texture_id); 591 } 592 593 void VideoDecoderShim::FlushCommandBuffer() { 594 context_provider_->ContextGL()->Flush(); 595 } 596 597 } // namespace content 598