1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "media/filters/video_renderer_impl.h" 6 7 #include "base/bind.h" 8 #include "base/callback.h" 9 #include "base/callback_helpers.h" 10 #include "base/debug/trace_event.h" 11 #include "base/location.h" 12 #include "base/single_thread_task_runner.h" 13 #include "base/threading/platform_thread.h" 14 #include "media/base/bind_to_current_loop.h" 15 #include "media/base/buffers.h" 16 #include "media/base/limits.h" 17 #include "media/base/pipeline.h" 18 #include "media/base/video_frame.h" 19 20 namespace media { 21 22 VideoRendererImpl::VideoRendererImpl( 23 const scoped_refptr<base::SingleThreadTaskRunner>& task_runner, 24 ScopedVector<VideoDecoder> decoders, 25 const SetDecryptorReadyCB& set_decryptor_ready_cb, 26 const PaintCB& paint_cb, 27 bool drop_frames, 28 const scoped_refptr<MediaLog>& media_log) 29 : task_runner_(task_runner), 30 video_frame_stream_(new VideoFrameStream(task_runner, 31 decoders.Pass(), 32 set_decryptor_ready_cb, 33 media_log)), 34 low_delay_(false), 35 received_end_of_stream_(false), 36 rendered_end_of_stream_(false), 37 frame_available_(&lock_), 38 state_(kUninitialized), 39 thread_(), 40 pending_read_(false), 41 drop_frames_(drop_frames), 42 buffering_state_(BUFFERING_HAVE_NOTHING), 43 paint_cb_(paint_cb), 44 last_timestamp_(kNoTimestamp()), 45 last_painted_timestamp_(kNoTimestamp()), 46 frames_decoded_(0), 47 frames_dropped_(0), 48 is_shutting_down_(false), 49 weak_factory_(this) { 50 DCHECK(!paint_cb_.is_null()); 51 } 52 53 VideoRendererImpl::~VideoRendererImpl() { 54 DCHECK(task_runner_->BelongsToCurrentThread()); 55 56 { 57 base::AutoLock auto_lock(lock_); 58 is_shutting_down_ = true; 59 frame_available_.Signal(); 60 } 61 62 if (!thread_.is_null()) 63 base::PlatformThread::Join(thread_); 64 65 if (!init_cb_.is_null()) 66 base::ResetAndReturn(&init_cb_).Run(PIPELINE_ERROR_ABORT); 67 68 if (!flush_cb_.is_null()) 69 base::ResetAndReturn(&flush_cb_).Run(); 70 } 71 72 void VideoRendererImpl::Flush(const base::Closure& callback) { 73 DVLOG(1) << __FUNCTION__; 74 DCHECK(task_runner_->BelongsToCurrentThread()); 75 base::AutoLock auto_lock(lock_); 76 DCHECK_EQ(state_, kPlaying); 77 flush_cb_ = callback; 78 state_ = kFlushing; 79 80 // This is necessary if the |video_frame_stream_| has already seen an end of 81 // stream and needs to drain it before flushing it. 82 ready_frames_.clear(); 83 if (buffering_state_ != BUFFERING_HAVE_NOTHING) { 84 buffering_state_ = BUFFERING_HAVE_NOTHING; 85 buffering_state_cb_.Run(BUFFERING_HAVE_NOTHING); 86 } 87 received_end_of_stream_ = false; 88 rendered_end_of_stream_ = false; 89 90 video_frame_stream_->Reset( 91 base::Bind(&VideoRendererImpl::OnVideoFrameStreamResetDone, 92 weak_factory_.GetWeakPtr())); 93 } 94 95 void VideoRendererImpl::StartPlayingFrom(base::TimeDelta timestamp) { 96 DVLOG(1) << __FUNCTION__ << "(" << timestamp.InMicroseconds() << ")"; 97 DCHECK(task_runner_->BelongsToCurrentThread()); 98 base::AutoLock auto_lock(lock_); 99 DCHECK_EQ(state_, kFlushed); 100 DCHECK(!pending_read_); 101 DCHECK(ready_frames_.empty()); 102 DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING); 103 104 state_ = kPlaying; 105 start_timestamp_ = timestamp; 106 AttemptRead_Locked(); 107 } 108 109 void VideoRendererImpl::Initialize(DemuxerStream* stream, 110 bool low_delay, 111 const PipelineStatusCB& init_cb, 112 const StatisticsCB& statistics_cb, 113 const BufferingStateCB& buffering_state_cb, 114 const base::Closure& ended_cb, 115 const PipelineStatusCB& error_cb, 116 const TimeDeltaCB& get_time_cb) { 117 DCHECK(task_runner_->BelongsToCurrentThread()); 118 base::AutoLock auto_lock(lock_); 119 DCHECK(stream); 120 DCHECK_EQ(stream->type(), DemuxerStream::VIDEO); 121 DCHECK(!init_cb.is_null()); 122 DCHECK(!statistics_cb.is_null()); 123 DCHECK(!buffering_state_cb.is_null()); 124 DCHECK(!ended_cb.is_null()); 125 DCHECK(!get_time_cb.is_null()); 126 DCHECK_EQ(kUninitialized, state_); 127 128 low_delay_ = low_delay; 129 130 // Always post |init_cb_| because |this| could be destroyed if initialization 131 // failed. 132 init_cb_ = BindToCurrentLoop(init_cb); 133 134 statistics_cb_ = statistics_cb; 135 buffering_state_cb_ = buffering_state_cb; 136 ended_cb_ = ended_cb; 137 error_cb_ = error_cb; 138 get_time_cb_ = get_time_cb; 139 state_ = kInitializing; 140 141 video_frame_stream_->Initialize( 142 stream, 143 low_delay, 144 statistics_cb, 145 base::Bind(&VideoRendererImpl::OnVideoFrameStreamInitialized, 146 weak_factory_.GetWeakPtr())); 147 } 148 149 void VideoRendererImpl::OnVideoFrameStreamInitialized(bool success) { 150 DCHECK(task_runner_->BelongsToCurrentThread()); 151 base::AutoLock auto_lock(lock_); 152 DCHECK_EQ(state_, kInitializing); 153 154 if (!success) { 155 state_ = kUninitialized; 156 base::ResetAndReturn(&init_cb_).Run(DECODER_ERROR_NOT_SUPPORTED); 157 return; 158 } 159 160 // We're all good! Consider ourselves flushed. (ThreadMain() should never 161 // see us in the kUninitialized state). 162 // Since we had an initial Preroll(), we consider ourself flushed, because we 163 // have not populated any buffers yet. 164 state_ = kFlushed; 165 166 // Create our video thread. 167 CHECK(base::PlatformThread::Create(0, this, &thread_)); 168 169 #if defined(OS_WIN) 170 // Bump up our priority so our sleeping is more accurate. 171 // TODO(scherkus): find out if this is necessary, but it seems to help. 172 ::SetThreadPriority(thread_.platform_handle(), THREAD_PRIORITY_ABOVE_NORMAL); 173 #endif // defined(OS_WIN) 174 base::ResetAndReturn(&init_cb_).Run(PIPELINE_OK); 175 } 176 177 // PlatformThread::Delegate implementation. 178 void VideoRendererImpl::ThreadMain() { 179 base::PlatformThread::SetName("CrVideoRenderer"); 180 181 // The number of milliseconds to idle when we do not have anything to do. 182 // Nothing special about the value, other than we're being more OS-friendly 183 // than sleeping for 1 millisecond. 184 // 185 // TODO(scherkus): switch to pure event-driven frame timing instead of this 186 // kIdleTimeDelta business http://crbug.com/106874 187 const base::TimeDelta kIdleTimeDelta = 188 base::TimeDelta::FromMilliseconds(10); 189 190 // If we have no frames and haven't painted any frame for certain amount of 191 // time, declare BUFFERING_HAVE_NOTHING. 192 const base::TimeDelta kTimeToDeclareHaveNothing = 193 base::TimeDelta::FromSeconds(3); 194 195 for (;;) { 196 base::AutoLock auto_lock(lock_); 197 198 // Thread exit condition. 199 if (is_shutting_down_) 200 return; 201 202 // Remain idle as long as we're not playing. 203 if (state_ != kPlaying || buffering_state_ != BUFFERING_HAVE_ENOUGH) { 204 UpdateStatsAndWait_Locked(kIdleTimeDelta); 205 continue; 206 } 207 208 base::TimeDelta now = get_time_cb_.Run(); 209 210 // Remain idle until we have the next frame ready for rendering. 211 if (ready_frames_.empty()) { 212 if (received_end_of_stream_) { 213 if (!rendered_end_of_stream_) { 214 rendered_end_of_stream_ = true; 215 task_runner_->PostTask(FROM_HERE, ended_cb_); 216 } 217 } else if (last_painted_timestamp_ != kNoTimestamp() && 218 now - last_painted_timestamp_ >= kTimeToDeclareHaveNothing) { 219 buffering_state_ = BUFFERING_HAVE_NOTHING; 220 task_runner_->PostTask( 221 FROM_HERE, base::Bind(buffering_state_cb_, BUFFERING_HAVE_NOTHING)); 222 } 223 224 UpdateStatsAndWait_Locked(kIdleTimeDelta); 225 continue; 226 } 227 228 base::TimeDelta target_paint_timestamp = ready_frames_.front()->timestamp(); 229 base::TimeDelta latest_paint_timestamp; 230 231 // Deadline is defined as the duration between this frame and the next 232 // frame, using the delta between this frame and the previous frame as the 233 // assumption for frame duration. 234 // 235 // TODO(scherkus): This can be vastly improved. Use a histogram to measure 236 // the accuracy of our frame timing code. http://crbug.com/149829 237 if (last_timestamp_ == kNoTimestamp()) { 238 latest_paint_timestamp = base::TimeDelta::Max(); 239 } else { 240 base::TimeDelta duration = target_paint_timestamp - last_timestamp_; 241 latest_paint_timestamp = target_paint_timestamp + duration; 242 } 243 244 // Remain idle until we've reached our target paint window. 245 if (now < target_paint_timestamp) { 246 UpdateStatsAndWait_Locked(kIdleTimeDelta); 247 continue; 248 } 249 250 if (now > latest_paint_timestamp && drop_frames_) { 251 DropNextReadyFrame_Locked(); 252 continue; 253 } 254 255 // Congratulations! You've made it past the video frame timing gauntlet. 256 // 257 // At this point enough time has passed that the next frame that ready for 258 // rendering. 259 PaintNextReadyFrame_Locked(); 260 } 261 } 262 263 void VideoRendererImpl::PaintNextReadyFrame_Locked() { 264 lock_.AssertAcquired(); 265 266 scoped_refptr<VideoFrame> next_frame = ready_frames_.front(); 267 ready_frames_.pop_front(); 268 frames_decoded_++; 269 270 last_timestamp_ = next_frame->timestamp(); 271 last_painted_timestamp_ = next_frame->timestamp(); 272 273 paint_cb_.Run(next_frame); 274 275 task_runner_->PostTask( 276 FROM_HERE, 277 base::Bind(&VideoRendererImpl::AttemptRead, weak_factory_.GetWeakPtr())); 278 } 279 280 void VideoRendererImpl::DropNextReadyFrame_Locked() { 281 TRACE_EVENT0("media", "VideoRendererImpl:frameDropped"); 282 283 lock_.AssertAcquired(); 284 285 last_timestamp_ = ready_frames_.front()->timestamp(); 286 ready_frames_.pop_front(); 287 frames_decoded_++; 288 frames_dropped_++; 289 290 task_runner_->PostTask( 291 FROM_HERE, 292 base::Bind(&VideoRendererImpl::AttemptRead, weak_factory_.GetWeakPtr())); 293 } 294 295 void VideoRendererImpl::FrameReady(VideoFrameStream::Status status, 296 const scoped_refptr<VideoFrame>& frame) { 297 DCHECK(task_runner_->BelongsToCurrentThread()); 298 base::AutoLock auto_lock(lock_); 299 DCHECK_NE(state_, kUninitialized); 300 DCHECK_NE(state_, kFlushed); 301 302 CHECK(pending_read_); 303 pending_read_ = false; 304 305 if (status == VideoFrameStream::DECODE_ERROR || 306 status == VideoFrameStream::DECRYPT_ERROR) { 307 DCHECK(!frame.get()); 308 PipelineStatus error = PIPELINE_ERROR_DECODE; 309 if (status == VideoFrameStream::DECRYPT_ERROR) 310 error = PIPELINE_ERROR_DECRYPT; 311 task_runner_->PostTask(FROM_HERE, base::Bind(error_cb_, error)); 312 return; 313 } 314 315 // Already-queued VideoFrameStream ReadCB's can fire after various state 316 // transitions have happened; in that case just drop those frames immediately. 317 if (state_ == kFlushing) 318 return; 319 320 DCHECK_EQ(state_, kPlaying); 321 322 // Can happen when demuxers are preparing for a new Seek(). 323 if (!frame.get()) { 324 DCHECK_EQ(status, VideoFrameStream::DEMUXER_READ_ABORTED); 325 return; 326 } 327 328 if (frame->end_of_stream()) { 329 DCHECK(!received_end_of_stream_); 330 received_end_of_stream_ = true; 331 } else { 332 // Maintain the latest frame decoded so the correct frame is displayed after 333 // prerolling has completed. 334 if (frame->timestamp() <= start_timestamp_) 335 ready_frames_.clear(); 336 AddReadyFrame_Locked(frame); 337 } 338 339 // Signal buffering state if we've met our conditions for having enough data. 340 if (buffering_state_ != BUFFERING_HAVE_ENOUGH && HaveEnoughData_Locked()) 341 TransitionToHaveEnough_Locked(); 342 343 // Always request more decoded video if we have capacity. This serves two 344 // purposes: 345 // 1) Prerolling while paused 346 // 2) Keeps decoding going if video rendering thread starts falling behind 347 AttemptRead_Locked(); 348 } 349 350 bool VideoRendererImpl::HaveEnoughData_Locked() { 351 DCHECK_EQ(state_, kPlaying); 352 return received_end_of_stream_ || 353 !video_frame_stream_->CanReadWithoutStalling() || 354 ready_frames_.size() >= static_cast<size_t>(limits::kMaxVideoFrames) || 355 (low_delay_ && ready_frames_.size() > 0); 356 } 357 358 void VideoRendererImpl::TransitionToHaveEnough_Locked() { 359 DCHECK(task_runner_->BelongsToCurrentThread()); 360 DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING); 361 362 if (!ready_frames_.empty()) { 363 // Because the clock might remain paused in for an undetermined amount 364 // of time (e.g., seeking while paused), paint the first frame. 365 PaintNextReadyFrame_Locked(); 366 } 367 368 buffering_state_ = BUFFERING_HAVE_ENOUGH; 369 buffering_state_cb_.Run(BUFFERING_HAVE_ENOUGH); 370 } 371 372 void VideoRendererImpl::AddReadyFrame_Locked( 373 const scoped_refptr<VideoFrame>& frame) { 374 DCHECK(task_runner_->BelongsToCurrentThread()); 375 lock_.AssertAcquired(); 376 DCHECK(!frame->end_of_stream()); 377 378 ready_frames_.push_back(frame); 379 DCHECK_LE(ready_frames_.size(), 380 static_cast<size_t>(limits::kMaxVideoFrames)); 381 382 // Avoid needlessly waking up |thread_| unless playing. 383 if (state_ == kPlaying) 384 frame_available_.Signal(); 385 } 386 387 void VideoRendererImpl::AttemptRead() { 388 base::AutoLock auto_lock(lock_); 389 AttemptRead_Locked(); 390 } 391 392 void VideoRendererImpl::AttemptRead_Locked() { 393 DCHECK(task_runner_->BelongsToCurrentThread()); 394 lock_.AssertAcquired(); 395 396 if (pending_read_ || received_end_of_stream_ || 397 ready_frames_.size() == static_cast<size_t>(limits::kMaxVideoFrames)) { 398 return; 399 } 400 401 switch (state_) { 402 case kPlaying: 403 pending_read_ = true; 404 video_frame_stream_->Read(base::Bind(&VideoRendererImpl::FrameReady, 405 weak_factory_.GetWeakPtr())); 406 return; 407 408 case kUninitialized: 409 case kInitializing: 410 case kFlushing: 411 case kFlushed: 412 return; 413 } 414 } 415 416 void VideoRendererImpl::OnVideoFrameStreamResetDone() { 417 base::AutoLock auto_lock(lock_); 418 DCHECK_EQ(kFlushing, state_); 419 DCHECK(!pending_read_); 420 DCHECK(ready_frames_.empty()); 421 DCHECK(!received_end_of_stream_); 422 DCHECK(!rendered_end_of_stream_); 423 DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING); 424 425 state_ = kFlushed; 426 last_timestamp_ = kNoTimestamp(); 427 last_painted_timestamp_ = kNoTimestamp(); 428 base::ResetAndReturn(&flush_cb_).Run(); 429 } 430 431 void VideoRendererImpl::UpdateStatsAndWait_Locked( 432 base::TimeDelta wait_duration) { 433 lock_.AssertAcquired(); 434 DCHECK_GE(frames_decoded_, 0); 435 DCHECK_LE(frames_dropped_, frames_decoded_); 436 437 if (frames_decoded_) { 438 PipelineStatistics statistics; 439 statistics.video_frames_decoded = frames_decoded_; 440 statistics.video_frames_dropped = frames_dropped_; 441 task_runner_->PostTask(FROM_HERE, base::Bind(statistics_cb_, statistics)); 442 443 frames_decoded_ = 0; 444 frames_dropped_ = 0; 445 } 446 447 frame_available_.TimedWait(wait_duration); 448 } 449 450 } // namespace media 451