Home | History | Annotate | Download | only in filters 
      1 // Copyright 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "media/filters/video_renderer_impl.h"
      6 
      7 #include "base/bind.h"
      8 #include "base/callback.h"
      9 #include "base/callback_helpers.h"
     10 #include "base/debug/trace_event.h"
     11 #include "base/location.h"
     12 #include "base/single_thread_task_runner.h"
     13 #include "base/threading/platform_thread.h"
     14 #include "media/base/bind_to_current_loop.h"
     15 #include "media/base/buffers.h"
     16 #include "media/base/limits.h"
     17 #include "media/base/pipeline.h"
     18 #include "media/base/video_frame.h"
     19 
     20 namespace media {
     21 
     22 VideoRendererImpl::VideoRendererImpl(
     23     const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
     24     ScopedVector<VideoDecoder> decoders,
     25     const SetDecryptorReadyCB& set_decryptor_ready_cb,
     26     const PaintCB& paint_cb,
     27     bool drop_frames,
     28     const scoped_refptr<MediaLog>& media_log)
     29     : task_runner_(task_runner),
     30       video_frame_stream_(new VideoFrameStream(task_runner,
     31                                                decoders.Pass(),
     32                                                set_decryptor_ready_cb,
     33                                                media_log)),
     34       low_delay_(false),
     35       received_end_of_stream_(false),
     36       rendered_end_of_stream_(false),
     37       frame_available_(&lock_),
     38       state_(kUninitialized),
     39       thread_(),
     40       pending_read_(false),
     41       drop_frames_(drop_frames),
     42       buffering_state_(BUFFERING_HAVE_NOTHING),
     43       paint_cb_(paint_cb),
     44       last_timestamp_(kNoTimestamp()),
     45       last_painted_timestamp_(kNoTimestamp()),
     46       frames_decoded_(0),
     47       frames_dropped_(0),
     48       is_shutting_down_(false),
     49       weak_factory_(this) {
     50   DCHECK(!paint_cb_.is_null());
     51 }
     52 
     53 VideoRendererImpl::~VideoRendererImpl() {
     54   DCHECK(task_runner_->BelongsToCurrentThread());
     55 
     56   {
     57     base::AutoLock auto_lock(lock_);
     58     is_shutting_down_ = true;
     59     frame_available_.Signal();
     60   }
     61 
     62   if (!thread_.is_null())
     63     base::PlatformThread::Join(thread_);
     64 
     65   if (!init_cb_.is_null())
     66     base::ResetAndReturn(&init_cb_).Run(PIPELINE_ERROR_ABORT);
     67 
     68   if (!flush_cb_.is_null())
     69     base::ResetAndReturn(&flush_cb_).Run();
     70 }
     71 
     72 void VideoRendererImpl::Flush(const base::Closure& callback) {
     73   DVLOG(1) << __FUNCTION__;
     74   DCHECK(task_runner_->BelongsToCurrentThread());
     75   base::AutoLock auto_lock(lock_);
     76   DCHECK_EQ(state_, kPlaying);
     77   flush_cb_ = callback;
     78   state_ = kFlushing;
     79 
     80   // This is necessary if the |video_frame_stream_| has already seen an end of
     81   // stream and needs to drain it before flushing it.
     82   ready_frames_.clear();
     83   if (buffering_state_ != BUFFERING_HAVE_NOTHING) {
     84     buffering_state_ = BUFFERING_HAVE_NOTHING;
     85     buffering_state_cb_.Run(BUFFERING_HAVE_NOTHING);
     86   }
     87   received_end_of_stream_ = false;
     88   rendered_end_of_stream_ = false;
     89 
     90   video_frame_stream_->Reset(
     91       base::Bind(&VideoRendererImpl::OnVideoFrameStreamResetDone,
     92                  weak_factory_.GetWeakPtr()));
     93 }
     94 
     95 void VideoRendererImpl::StartPlayingFrom(base::TimeDelta timestamp) {
     96   DVLOG(1) << __FUNCTION__ << "(" << timestamp.InMicroseconds() << ")";
     97   DCHECK(task_runner_->BelongsToCurrentThread());
     98   base::AutoLock auto_lock(lock_);
     99   DCHECK_EQ(state_, kFlushed);
    100   DCHECK(!pending_read_);
    101   DCHECK(ready_frames_.empty());
    102   DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING);
    103 
    104   state_ = kPlaying;
    105   start_timestamp_ = timestamp;
    106   AttemptRead_Locked();
    107 }
    108 
    109 void VideoRendererImpl::Initialize(DemuxerStream* stream,
    110                                    bool low_delay,
    111                                    const PipelineStatusCB& init_cb,
    112                                    const StatisticsCB& statistics_cb,
    113                                    const BufferingStateCB& buffering_state_cb,
    114                                    const base::Closure& ended_cb,
    115                                    const PipelineStatusCB& error_cb,
    116                                    const TimeDeltaCB& get_time_cb) {
    117   DCHECK(task_runner_->BelongsToCurrentThread());
    118   base::AutoLock auto_lock(lock_);
    119   DCHECK(stream);
    120   DCHECK_EQ(stream->type(), DemuxerStream::VIDEO);
    121   DCHECK(!init_cb.is_null());
    122   DCHECK(!statistics_cb.is_null());
    123   DCHECK(!buffering_state_cb.is_null());
    124   DCHECK(!ended_cb.is_null());
    125   DCHECK(!get_time_cb.is_null());
    126   DCHECK_EQ(kUninitialized, state_);
    127 
    128   low_delay_ = low_delay;
    129 
    130   // Always post |init_cb_| because |this| could be destroyed if initialization
    131   // failed.
    132   init_cb_ = BindToCurrentLoop(init_cb);
    133 
    134   statistics_cb_ = statistics_cb;
    135   buffering_state_cb_ = buffering_state_cb;
    136   ended_cb_ = ended_cb;
    137   error_cb_ = error_cb;
    138   get_time_cb_ = get_time_cb;
    139   state_ = kInitializing;
    140 
    141   video_frame_stream_->Initialize(
    142       stream,
    143       low_delay,
    144       statistics_cb,
    145       base::Bind(&VideoRendererImpl::OnVideoFrameStreamInitialized,
    146                  weak_factory_.GetWeakPtr()));
    147 }
    148 
    149 void VideoRendererImpl::OnVideoFrameStreamInitialized(bool success) {
    150   DCHECK(task_runner_->BelongsToCurrentThread());
    151   base::AutoLock auto_lock(lock_);
    152   DCHECK_EQ(state_, kInitializing);
    153 
    154   if (!success) {
    155     state_ = kUninitialized;
    156     base::ResetAndReturn(&init_cb_).Run(DECODER_ERROR_NOT_SUPPORTED);
    157     return;
    158   }
    159 
    160   // We're all good!  Consider ourselves flushed. (ThreadMain() should never
    161   // see us in the kUninitialized state).
    162   // Since we had an initial Preroll(), we consider ourself flushed, because we
    163   // have not populated any buffers yet.
    164   state_ = kFlushed;
    165 
    166   // Create our video thread.
    167   CHECK(base::PlatformThread::Create(0, this, &thread_));
    168 
    169 #if defined(OS_WIN)
    170   // Bump up our priority so our sleeping is more accurate.
    171   // TODO(scherkus): find out if this is necessary, but it seems to help.
    172   ::SetThreadPriority(thread_.platform_handle(), THREAD_PRIORITY_ABOVE_NORMAL);
    173 #endif  // defined(OS_WIN)
    174   base::ResetAndReturn(&init_cb_).Run(PIPELINE_OK);
    175 }
    176 
    177 // PlatformThread::Delegate implementation.
    178 void VideoRendererImpl::ThreadMain() {
    179   base::PlatformThread::SetName("CrVideoRenderer");
    180 
    181   // The number of milliseconds to idle when we do not have anything to do.
    182   // Nothing special about the value, other than we're being more OS-friendly
    183   // than sleeping for 1 millisecond.
    184   //
    185   // TODO(scherkus): switch to pure event-driven frame timing instead of this
    186   // kIdleTimeDelta business http://crbug.com/106874
    187   const base::TimeDelta kIdleTimeDelta =
    188       base::TimeDelta::FromMilliseconds(10);
    189 
    190   // If we have no frames and haven't painted any frame for certain amount of
    191   // time, declare BUFFERING_HAVE_NOTHING.
    192   const base::TimeDelta kTimeToDeclareHaveNothing =
    193       base::TimeDelta::FromSeconds(3);
    194 
    195   for (;;) {
    196     base::AutoLock auto_lock(lock_);
    197 
    198     // Thread exit condition.
    199     if (is_shutting_down_)
    200       return;
    201 
    202     // Remain idle as long as we're not playing.
    203     if (state_ != kPlaying || buffering_state_ != BUFFERING_HAVE_ENOUGH) {
    204       UpdateStatsAndWait_Locked(kIdleTimeDelta);
    205       continue;
    206     }
    207 
    208     base::TimeDelta now = get_time_cb_.Run();
    209 
    210     // Remain idle until we have the next frame ready for rendering.
    211     if (ready_frames_.empty()) {
    212       if (received_end_of_stream_) {
    213         if (!rendered_end_of_stream_) {
    214           rendered_end_of_stream_ = true;
    215           task_runner_->PostTask(FROM_HERE, ended_cb_);
    216         }
    217       } else if (last_painted_timestamp_ != kNoTimestamp() &&
    218                  now - last_painted_timestamp_ >= kTimeToDeclareHaveNothing) {
    219         buffering_state_ = BUFFERING_HAVE_NOTHING;
    220         task_runner_->PostTask(
    221             FROM_HERE, base::Bind(buffering_state_cb_, BUFFERING_HAVE_NOTHING));
    222       }
    223 
    224       UpdateStatsAndWait_Locked(kIdleTimeDelta);
    225       continue;
    226     }
    227 
    228     base::TimeDelta target_paint_timestamp = ready_frames_.front()->timestamp();
    229     base::TimeDelta latest_paint_timestamp;
    230 
    231     // Deadline is defined as the duration between this frame and the next
    232     // frame, using the delta between this frame and the previous frame as the
    233     // assumption for frame duration.
    234     //
    235     // TODO(scherkus): This can be vastly improved. Use a histogram to measure
    236     // the accuracy of our frame timing code. http://crbug.com/149829
    237     if (last_timestamp_ == kNoTimestamp()) {
    238       latest_paint_timestamp = base::TimeDelta::Max();
    239     } else {
    240       base::TimeDelta duration = target_paint_timestamp - last_timestamp_;
    241       latest_paint_timestamp = target_paint_timestamp + duration;
    242     }
    243 
    244     // Remain idle until we've reached our target paint window.
    245     if (now < target_paint_timestamp) {
    246       UpdateStatsAndWait_Locked(kIdleTimeDelta);
    247       continue;
    248     }
    249 
    250     if (now > latest_paint_timestamp && drop_frames_) {
    251       DropNextReadyFrame_Locked();
    252       continue;
    253     }
    254 
    255     // Congratulations! You've made it past the video frame timing gauntlet.
    256     //
    257     // At this point enough time has passed that the next frame that ready for
    258     // rendering.
    259     PaintNextReadyFrame_Locked();
    260   }
    261 }
    262 
    263 void VideoRendererImpl::PaintNextReadyFrame_Locked() {
    264   lock_.AssertAcquired();
    265 
    266   scoped_refptr<VideoFrame> next_frame = ready_frames_.front();
    267   ready_frames_.pop_front();
    268   frames_decoded_++;
    269 
    270   last_timestamp_ = next_frame->timestamp();
    271   last_painted_timestamp_ = next_frame->timestamp();
    272 
    273   paint_cb_.Run(next_frame);
    274 
    275   task_runner_->PostTask(
    276       FROM_HERE,
    277       base::Bind(&VideoRendererImpl::AttemptRead, weak_factory_.GetWeakPtr()));
    278 }
    279 
    280 void VideoRendererImpl::DropNextReadyFrame_Locked() {
    281   TRACE_EVENT0("media", "VideoRendererImpl:frameDropped");
    282 
    283   lock_.AssertAcquired();
    284 
    285   last_timestamp_ = ready_frames_.front()->timestamp();
    286   ready_frames_.pop_front();
    287   frames_decoded_++;
    288   frames_dropped_++;
    289 
    290   task_runner_->PostTask(
    291       FROM_HERE,
    292       base::Bind(&VideoRendererImpl::AttemptRead, weak_factory_.GetWeakPtr()));
    293 }
    294 
    295 void VideoRendererImpl::FrameReady(VideoFrameStream::Status status,
    296                                    const scoped_refptr<VideoFrame>& frame) {
    297   DCHECK(task_runner_->BelongsToCurrentThread());
    298   base::AutoLock auto_lock(lock_);
    299   DCHECK_NE(state_, kUninitialized);
    300   DCHECK_NE(state_, kFlushed);
    301 
    302   CHECK(pending_read_);
    303   pending_read_ = false;
    304 
    305   if (status == VideoFrameStream::DECODE_ERROR ||
    306       status == VideoFrameStream::DECRYPT_ERROR) {
    307     DCHECK(!frame.get());
    308     PipelineStatus error = PIPELINE_ERROR_DECODE;
    309     if (status == VideoFrameStream::DECRYPT_ERROR)
    310       error = PIPELINE_ERROR_DECRYPT;
    311     task_runner_->PostTask(FROM_HERE, base::Bind(error_cb_, error));
    312     return;
    313   }
    314 
    315   // Already-queued VideoFrameStream ReadCB's can fire after various state
    316   // transitions have happened; in that case just drop those frames immediately.
    317   if (state_ == kFlushing)
    318     return;
    319 
    320   DCHECK_EQ(state_, kPlaying);
    321 
    322   // Can happen when demuxers are preparing for a new Seek().
    323   if (!frame.get()) {
    324     DCHECK_EQ(status, VideoFrameStream::DEMUXER_READ_ABORTED);
    325     return;
    326   }
    327 
    328   if (frame->end_of_stream()) {
    329     DCHECK(!received_end_of_stream_);
    330     received_end_of_stream_ = true;
    331   } else {
    332     // Maintain the latest frame decoded so the correct frame is displayed after
    333     // prerolling has completed.
    334     if (frame->timestamp() <= start_timestamp_)
    335       ready_frames_.clear();
    336     AddReadyFrame_Locked(frame);
    337   }
    338 
    339   // Signal buffering state if we've met our conditions for having enough data.
    340   if (buffering_state_ != BUFFERING_HAVE_ENOUGH && HaveEnoughData_Locked())
    341     TransitionToHaveEnough_Locked();
    342 
    343   // Always request more decoded video if we have capacity. This serves two
    344   // purposes:
    345   //   1) Prerolling while paused
    346   //   2) Keeps decoding going if video rendering thread starts falling behind
    347   AttemptRead_Locked();
    348 }
    349 
    350 bool VideoRendererImpl::HaveEnoughData_Locked() {
    351   DCHECK_EQ(state_, kPlaying);
    352   return received_end_of_stream_ ||
    353       !video_frame_stream_->CanReadWithoutStalling() ||
    354       ready_frames_.size() >= static_cast<size_t>(limits::kMaxVideoFrames) ||
    355       (low_delay_ && ready_frames_.size() > 0);
    356 }
    357 
    358 void VideoRendererImpl::TransitionToHaveEnough_Locked() {
    359   DCHECK(task_runner_->BelongsToCurrentThread());
    360   DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING);
    361 
    362   if (!ready_frames_.empty()) {
    363     // Because the clock might remain paused in for an undetermined amount
    364     // of time (e.g., seeking while paused), paint the first frame.
    365     PaintNextReadyFrame_Locked();
    366   }
    367 
    368   buffering_state_ = BUFFERING_HAVE_ENOUGH;
    369   buffering_state_cb_.Run(BUFFERING_HAVE_ENOUGH);
    370 }
    371 
    372 void VideoRendererImpl::AddReadyFrame_Locked(
    373     const scoped_refptr<VideoFrame>& frame) {
    374   DCHECK(task_runner_->BelongsToCurrentThread());
    375   lock_.AssertAcquired();
    376   DCHECK(!frame->end_of_stream());
    377 
    378   ready_frames_.push_back(frame);
    379   DCHECK_LE(ready_frames_.size(),
    380             static_cast<size_t>(limits::kMaxVideoFrames));
    381 
    382   // Avoid needlessly waking up |thread_| unless playing.
    383   if (state_ == kPlaying)
    384     frame_available_.Signal();
    385 }
    386 
    387 void VideoRendererImpl::AttemptRead() {
    388   base::AutoLock auto_lock(lock_);
    389   AttemptRead_Locked();
    390 }
    391 
    392 void VideoRendererImpl::AttemptRead_Locked() {
    393   DCHECK(task_runner_->BelongsToCurrentThread());
    394   lock_.AssertAcquired();
    395 
    396   if (pending_read_ || received_end_of_stream_ ||
    397       ready_frames_.size() == static_cast<size_t>(limits::kMaxVideoFrames)) {
    398     return;
    399   }
    400 
    401   switch (state_) {
    402     case kPlaying:
    403       pending_read_ = true;
    404       video_frame_stream_->Read(base::Bind(&VideoRendererImpl::FrameReady,
    405                                            weak_factory_.GetWeakPtr()));
    406       return;
    407 
    408     case kUninitialized:
    409     case kInitializing:
    410     case kFlushing:
    411     case kFlushed:
    412       return;
    413   }
    414 }
    415 
    416 void VideoRendererImpl::OnVideoFrameStreamResetDone() {
    417   base::AutoLock auto_lock(lock_);
    418   DCHECK_EQ(kFlushing, state_);
    419   DCHECK(!pending_read_);
    420   DCHECK(ready_frames_.empty());
    421   DCHECK(!received_end_of_stream_);
    422   DCHECK(!rendered_end_of_stream_);
    423   DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING);
    424 
    425   state_ = kFlushed;
    426   last_timestamp_ = kNoTimestamp();
    427   last_painted_timestamp_ = kNoTimestamp();
    428   base::ResetAndReturn(&flush_cb_).Run();
    429 }
    430 
    431 void VideoRendererImpl::UpdateStatsAndWait_Locked(
    432     base::TimeDelta wait_duration) {
    433   lock_.AssertAcquired();
    434   DCHECK_GE(frames_decoded_, 0);
    435   DCHECK_LE(frames_dropped_, frames_decoded_);
    436 
    437   if (frames_decoded_) {
    438     PipelineStatistics statistics;
    439     statistics.video_frames_decoded = frames_decoded_;
    440     statistics.video_frames_dropped = frames_dropped_;
    441     task_runner_->PostTask(FROM_HERE, base::Bind(statistics_cb_, statistics));
    442 
    443     frames_decoded_ = 0;
    444     frames_dropped_ = 0;
    445   }
    446 
    447   frame_available_.TimedWait(wait_duration);
    448 }
    449 
    450 }  // namespace media
    451