1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "media/filters/video_renderer_base.h" 6 7 #include "base/bind.h" 8 #include "base/callback.h" 9 #include "base/callback_helpers.h" 10 #include "base/message_loop/message_loop.h" 11 #include "base/threading/platform_thread.h" 12 #include "media/base/buffers.h" 13 #include "media/base/limits.h" 14 #include "media/base/pipeline.h" 15 #include "media/base/video_frame.h" 16 17 namespace media { 18 19 base::TimeDelta VideoRendererBase::kMaxLastFrameDuration() { 20 return base::TimeDelta::FromMilliseconds(250); 21 } 22 23 VideoRendererBase::VideoRendererBase( 24 const scoped_refptr<base::MessageLoopProxy>& message_loop, 25 ScopedVector<VideoDecoder> decoders, 26 const SetDecryptorReadyCB& set_decryptor_ready_cb, 27 const PaintCB& paint_cb, 28 const SetOpaqueCB& set_opaque_cb, 29 bool drop_frames) 30 : message_loop_(message_loop), 31 weak_factory_(this), 32 video_frame_stream_( 33 message_loop, decoders.Pass(), set_decryptor_ready_cb), 34 received_end_of_stream_(false), 35 frame_available_(&lock_), 36 state_(kUninitialized), 37 thread_(), 38 pending_read_(false), 39 drop_frames_(drop_frames), 40 playback_rate_(0), 41 paint_cb_(paint_cb), 42 set_opaque_cb_(set_opaque_cb), 43 last_timestamp_(kNoTimestamp()) { 44 DCHECK(!paint_cb_.is_null()); 45 } 46 47 VideoRendererBase::~VideoRendererBase() { 48 base::AutoLock auto_lock(lock_); 49 CHECK(state_ == kStopped || state_ == kUninitialized) << state_; 50 CHECK(thread_.is_null()); 51 } 52 53 void VideoRendererBase::Play(const base::Closure& callback) { 54 DCHECK(message_loop_->BelongsToCurrentThread()); 55 base::AutoLock auto_lock(lock_); 56 DCHECK_EQ(kPrerolled, state_); 57 state_ = kPlaying; 58 callback.Run(); 59 } 60 61 void VideoRendererBase::Pause(const base::Closure& callback) { 62 DCHECK(message_loop_->BelongsToCurrentThread()); 63 base::AutoLock auto_lock(lock_); 64 DCHECK(state_ != kUninitialized || state_ == kError); 65 state_ = kPaused; 66 callback.Run(); 67 } 68 69 void VideoRendererBase::Flush(const base::Closure& callback) { 70 DCHECK(message_loop_->BelongsToCurrentThread()); 71 base::AutoLock auto_lock(lock_); 72 DCHECK_EQ(state_, kPaused); 73 flush_cb_ = callback; 74 state_ = kFlushing; 75 76 // This is necessary if the |video_frame_stream_| has already seen an end of 77 // stream and needs to drain it before flushing it. 78 ready_frames_.clear(); 79 received_end_of_stream_ = false; 80 video_frame_stream_.Reset(base::Bind( 81 &VideoRendererBase::OnVideoFrameStreamResetDone, weak_this_)); 82 } 83 84 void VideoRendererBase::Stop(const base::Closure& callback) { 85 DCHECK(message_loop_->BelongsToCurrentThread()); 86 base::AutoLock auto_lock(lock_); 87 if (state_ == kUninitialized || state_ == kStopped) { 88 callback.Run(); 89 return; 90 } 91 92 // TODO(scherkus): Consider invalidating |weak_factory_| and replacing 93 // task-running guards that check |state_| with DCHECK(). 94 95 state_ = kStopped; 96 97 statistics_cb_.Reset(); 98 max_time_cb_.Reset(); 99 DoStopOrError_Locked(); 100 101 // Clean up our thread if present. 102 base::PlatformThreadHandle thread_to_join = base::PlatformThreadHandle(); 103 if (!thread_.is_null()) { 104 // Signal the thread since it's possible to get stopped with the video 105 // thread waiting for a read to complete. 106 frame_available_.Signal(); 107 std::swap(thread_, thread_to_join); 108 } 109 110 if (!thread_to_join.is_null()) { 111 base::AutoUnlock auto_unlock(lock_); 112 base::PlatformThread::Join(thread_to_join); 113 } 114 115 video_frame_stream_.Stop(callback); 116 } 117 118 void VideoRendererBase::SetPlaybackRate(float playback_rate) { 119 DCHECK(message_loop_->BelongsToCurrentThread()); 120 base::AutoLock auto_lock(lock_); 121 playback_rate_ = playback_rate; 122 } 123 124 void VideoRendererBase::Preroll(base::TimeDelta time, 125 const PipelineStatusCB& cb) { 126 DCHECK(message_loop_->BelongsToCurrentThread()); 127 base::AutoLock auto_lock(lock_); 128 DCHECK_EQ(state_, kFlushed) << "Must flush prior to prerolling."; 129 DCHECK(!cb.is_null()); 130 DCHECK(preroll_cb_.is_null()); 131 132 state_ = kPrerolling; 133 preroll_cb_ = cb; 134 preroll_timestamp_ = time; 135 AttemptRead_Locked(); 136 } 137 138 void VideoRendererBase::Initialize(DemuxerStream* stream, 139 const PipelineStatusCB& init_cb, 140 const StatisticsCB& statistics_cb, 141 const TimeCB& max_time_cb, 142 const NaturalSizeChangedCB& size_changed_cb, 143 const base::Closure& ended_cb, 144 const PipelineStatusCB& error_cb, 145 const TimeDeltaCB& get_time_cb, 146 const TimeDeltaCB& get_duration_cb) { 147 DCHECK(message_loop_->BelongsToCurrentThread()); 148 base::AutoLock auto_lock(lock_); 149 DCHECK(stream); 150 DCHECK_EQ(stream->type(), DemuxerStream::VIDEO); 151 DCHECK(!init_cb.is_null()); 152 DCHECK(!statistics_cb.is_null()); 153 DCHECK(!max_time_cb.is_null()); 154 DCHECK(!size_changed_cb.is_null()); 155 DCHECK(!ended_cb.is_null()); 156 DCHECK(!get_time_cb.is_null()); 157 DCHECK(!get_duration_cb.is_null()); 158 DCHECK_EQ(kUninitialized, state_); 159 160 weak_this_ = weak_factory_.GetWeakPtr(); 161 init_cb_ = init_cb; 162 statistics_cb_ = statistics_cb; 163 max_time_cb_ = max_time_cb; 164 size_changed_cb_ = size_changed_cb; 165 ended_cb_ = ended_cb; 166 error_cb_ = error_cb; 167 get_time_cb_ = get_time_cb; 168 get_duration_cb_ = get_duration_cb; 169 state_ = kInitializing; 170 171 video_frame_stream_.Initialize( 172 stream, 173 statistics_cb, 174 base::Bind(&VideoRendererBase::OnVideoFrameStreamInitialized, 175 weak_this_)); 176 } 177 178 void VideoRendererBase::OnVideoFrameStreamInitialized(bool success, 179 bool has_alpha) { 180 DCHECK(message_loop_->BelongsToCurrentThread()); 181 base::AutoLock auto_lock(lock_); 182 183 if (state_ == kStopped) 184 return; 185 186 DCHECK_EQ(state_, kInitializing); 187 188 if (!success) { 189 state_ = kUninitialized; 190 base::ResetAndReturn(&init_cb_).Run(DECODER_ERROR_NOT_SUPPORTED); 191 return; 192 } 193 194 // We're all good! Consider ourselves flushed. (ThreadMain() should never 195 // see us in the kUninitialized state). 196 // Since we had an initial Preroll(), we consider ourself flushed, because we 197 // have not populated any buffers yet. 198 state_ = kFlushed; 199 200 set_opaque_cb_.Run(!has_alpha); 201 set_opaque_cb_.Reset(); 202 203 // Create our video thread. 204 if (!base::PlatformThread::Create(0, this, &thread_)) { 205 NOTREACHED() << "Video thread creation failed"; 206 state_ = kError; 207 base::ResetAndReturn(&init_cb_).Run(PIPELINE_ERROR_INITIALIZATION_FAILED); 208 return; 209 } 210 211 #if defined(OS_WIN) 212 // Bump up our priority so our sleeping is more accurate. 213 // TODO(scherkus): find out if this is necessary, but it seems to help. 214 ::SetThreadPriority(thread_.platform_handle(), THREAD_PRIORITY_ABOVE_NORMAL); 215 #endif // defined(OS_WIN) 216 base::ResetAndReturn(&init_cb_).Run(PIPELINE_OK); 217 } 218 219 // PlatformThread::Delegate implementation. 220 void VideoRendererBase::ThreadMain() { 221 base::PlatformThread::SetName("CrVideoRenderer"); 222 223 // The number of milliseconds to idle when we do not have anything to do. 224 // Nothing special about the value, other than we're being more OS-friendly 225 // than sleeping for 1 millisecond. 226 // 227 // TODO(scherkus): switch to pure event-driven frame timing instead of this 228 // kIdleTimeDelta business http://crbug.com/106874 229 const base::TimeDelta kIdleTimeDelta = 230 base::TimeDelta::FromMilliseconds(10); 231 232 for (;;) { 233 base::AutoLock auto_lock(lock_); 234 235 // Thread exit condition. 236 if (state_ == kStopped) 237 return; 238 239 // Remain idle as long as we're not playing. 240 if (state_ != kPlaying || playback_rate_ == 0) { 241 frame_available_.TimedWait(kIdleTimeDelta); 242 continue; 243 } 244 245 // Remain idle until we have the next frame ready for rendering. 246 if (ready_frames_.empty()) { 247 if (received_end_of_stream_) { 248 state_ = kEnded; 249 ended_cb_.Run(); 250 251 // No need to sleep here as we idle when |state_ != kPlaying|. 252 continue; 253 } 254 255 frame_available_.TimedWait(kIdleTimeDelta); 256 continue; 257 } 258 259 base::TimeDelta remaining_time = 260 CalculateSleepDuration(ready_frames_.front(), playback_rate_); 261 262 // Sleep up to a maximum of our idle time until we're within the time to 263 // render the next frame. 264 if (remaining_time.InMicroseconds() > 0) { 265 remaining_time = std::min(remaining_time, kIdleTimeDelta); 266 frame_available_.TimedWait(remaining_time); 267 continue; 268 } 269 270 // Deadline is defined as the midpoint between this frame and the next 271 // frame, using the delta between this frame and the previous frame as the 272 // assumption for frame duration. 273 // 274 // TODO(scherkus): An improvement over midpoint might be selecting the 275 // minimum and/or maximum between the midpoint and some constants. As a 276 // thought experiment, consider what would be better than the midpoint 277 // for both the 1fps case and 120fps case. 278 // 279 // TODO(scherkus): This can be vastly improved. Use a histogram to measure 280 // the accuracy of our frame timing code. http://crbug.com/149829 281 if (drop_frames_ && last_timestamp_ != kNoTimestamp()) { 282 base::TimeDelta now = get_time_cb_.Run(); 283 base::TimeDelta deadline = ready_frames_.front()->GetTimestamp() + 284 (ready_frames_.front()->GetTimestamp() - last_timestamp_) / 2; 285 286 if (now > deadline) { 287 DropNextReadyFrame_Locked(); 288 continue; 289 } 290 } 291 292 // Congratulations! You've made it past the video frame timing gauntlet. 293 // 294 // At this point enough time has passed that the next frame that ready for 295 // rendering. 296 PaintNextReadyFrame_Locked(); 297 } 298 } 299 300 void VideoRendererBase::PaintNextReadyFrame_Locked() { 301 lock_.AssertAcquired(); 302 303 scoped_refptr<VideoFrame> next_frame = ready_frames_.front(); 304 ready_frames_.pop_front(); 305 306 last_timestamp_ = next_frame->GetTimestamp(); 307 308 const gfx::Size& natural_size = next_frame->natural_size(); 309 if (natural_size != last_natural_size_) { 310 last_natural_size_ = natural_size; 311 size_changed_cb_.Run(natural_size); 312 } 313 314 paint_cb_.Run(next_frame); 315 316 message_loop_->PostTask(FROM_HERE, base::Bind( 317 &VideoRendererBase::AttemptRead, weak_this_)); 318 } 319 320 void VideoRendererBase::DropNextReadyFrame_Locked() { 321 lock_.AssertAcquired(); 322 323 last_timestamp_ = ready_frames_.front()->GetTimestamp(); 324 ready_frames_.pop_front(); 325 326 PipelineStatistics statistics; 327 statistics.video_frames_dropped = 1; 328 statistics_cb_.Run(statistics); 329 330 message_loop_->PostTask(FROM_HERE, base::Bind( 331 &VideoRendererBase::AttemptRead, weak_this_)); 332 } 333 334 void VideoRendererBase::FrameReady(VideoFrameStream::Status status, 335 const scoped_refptr<VideoFrame>& frame) { 336 base::AutoLock auto_lock(lock_); 337 DCHECK_NE(state_, kUninitialized); 338 DCHECK_NE(state_, kFlushed); 339 340 CHECK(pending_read_); 341 pending_read_ = false; 342 343 if (status == VideoFrameStream::DECODE_ERROR || 344 status == VideoFrameStream::DECRYPT_ERROR) { 345 DCHECK(!frame.get()); 346 PipelineStatus error = PIPELINE_ERROR_DECODE; 347 if (status == VideoFrameStream::DECRYPT_ERROR) 348 error = PIPELINE_ERROR_DECRYPT; 349 350 if (!preroll_cb_.is_null()) { 351 base::ResetAndReturn(&preroll_cb_).Run(error); 352 return; 353 } 354 355 error_cb_.Run(error); 356 return; 357 } 358 359 // Already-queued VideoFrameStream ReadCB's can fire after various state 360 // transitions have happened; in that case just drop those frames immediately. 361 if (state_ == kStopped || state_ == kError || state_ == kFlushing) 362 return; 363 364 if (!frame.get()) { 365 // Abort preroll early for a NULL frame because we won't get more frames. 366 // A new preroll will be requested after this one completes so there is no 367 // point trying to collect more frames. 368 if (state_ == kPrerolling) 369 TransitionToPrerolled_Locked(); 370 371 return; 372 } 373 374 if (frame->IsEndOfStream()) { 375 DCHECK(!received_end_of_stream_); 376 received_end_of_stream_ = true; 377 max_time_cb_.Run(get_duration_cb_.Run()); 378 379 if (state_ == kPrerolling) 380 TransitionToPrerolled_Locked(); 381 382 return; 383 } 384 385 // Maintain the latest frame decoded so the correct frame is displayed after 386 // prerolling has completed. 387 if (state_ == kPrerolling && frame->GetTimestamp() <= preroll_timestamp_) { 388 ready_frames_.clear(); 389 } 390 391 AddReadyFrame_Locked(frame); 392 393 if (state_ == kPrerolling) { 394 if (!video_frame_stream_.CanReadWithoutStalling() || 395 ready_frames_.size() >= static_cast<size_t>(limits::kMaxVideoFrames)) { 396 TransitionToPrerolled_Locked(); 397 } 398 } else { 399 // We only count frames decoded during normal playback. 400 PipelineStatistics statistics; 401 statistics.video_frames_decoded = 1; 402 statistics_cb_.Run(statistics); 403 } 404 405 // Always request more decoded video if we have capacity. This serves two 406 // purposes: 407 // 1) Prerolling while paused 408 // 2) Keeps decoding going if video rendering thread starts falling behind 409 AttemptRead_Locked(); 410 } 411 412 void VideoRendererBase::AddReadyFrame_Locked( 413 const scoped_refptr<VideoFrame>& frame) { 414 lock_.AssertAcquired(); 415 DCHECK(!frame->IsEndOfStream()); 416 417 // Adjust the incoming frame if its rendering stop time is past the duration 418 // of the video itself. This is typically the last frame of the video and 419 // occurs if the container specifies a duration that isn't a multiple of the 420 // frame rate. Another way for this to happen is for the container to state 421 // a smaller duration than the largest packet timestamp. 422 base::TimeDelta duration = get_duration_cb_.Run(); 423 if (frame->GetTimestamp() > duration) { 424 frame->SetTimestamp(duration); 425 } 426 427 ready_frames_.push_back(frame); 428 DCHECK_LE(ready_frames_.size(), 429 static_cast<size_t>(limits::kMaxVideoFrames)); 430 431 max_time_cb_.Run(frame->GetTimestamp()); 432 433 // Avoid needlessly waking up |thread_| unless playing. 434 if (state_ == kPlaying) 435 frame_available_.Signal(); 436 } 437 438 void VideoRendererBase::AttemptRead() { 439 base::AutoLock auto_lock(lock_); 440 AttemptRead_Locked(); 441 } 442 443 void VideoRendererBase::AttemptRead_Locked() { 444 DCHECK(message_loop_->BelongsToCurrentThread()); 445 lock_.AssertAcquired(); 446 447 if (pending_read_ || received_end_of_stream_ || 448 ready_frames_.size() == static_cast<size_t>(limits::kMaxVideoFrames)) { 449 return; 450 } 451 452 switch (state_) { 453 case kPaused: 454 case kPrerolling: 455 case kPlaying: 456 pending_read_ = true; 457 video_frame_stream_.Read(base::Bind(&VideoRendererBase::FrameReady, 458 weak_this_)); 459 return; 460 461 case kUninitialized: 462 case kInitializing: 463 case kPrerolled: 464 case kFlushing: 465 case kFlushed: 466 case kEnded: 467 case kStopped: 468 case kError: 469 return; 470 } 471 } 472 473 void VideoRendererBase::OnVideoFrameStreamResetDone() { 474 base::AutoLock auto_lock(lock_); 475 if (state_ == kStopped) 476 return; 477 478 DCHECK_EQ(kFlushing, state_); 479 DCHECK(!pending_read_); 480 DCHECK(ready_frames_.empty()); 481 DCHECK(!received_end_of_stream_); 482 483 state_ = kFlushed; 484 last_timestamp_ = kNoTimestamp(); 485 base::ResetAndReturn(&flush_cb_).Run(); 486 } 487 488 base::TimeDelta VideoRendererBase::CalculateSleepDuration( 489 const scoped_refptr<VideoFrame>& next_frame, 490 float playback_rate) { 491 // Determine the current and next presentation timestamps. 492 base::TimeDelta now = get_time_cb_.Run(); 493 base::TimeDelta next_pts = next_frame->GetTimestamp(); 494 495 // Scale our sleep based on the playback rate. 496 base::TimeDelta sleep = next_pts - now; 497 return base::TimeDelta::FromMicroseconds( 498 static_cast<int64>(sleep.InMicroseconds() / playback_rate)); 499 } 500 501 void VideoRendererBase::DoStopOrError_Locked() { 502 lock_.AssertAcquired(); 503 last_timestamp_ = kNoTimestamp(); 504 ready_frames_.clear(); 505 } 506 507 void VideoRendererBase::TransitionToPrerolled_Locked() { 508 lock_.AssertAcquired(); 509 DCHECK_EQ(state_, kPrerolling); 510 511 state_ = kPrerolled; 512 513 // Because we might remain in the prerolled state for an undetermined amount 514 // of time (e.g., we seeked while paused), we'll paint the first prerolled 515 // frame. 516 if (!ready_frames_.empty()) 517 PaintNextReadyFrame_Locked(); 518 519 base::ResetAndReturn(&preroll_cb_).Run(PIPELINE_OK); 520 } 521 522 } // namespace media 523