Home | History | Annotate | Download | only in filters 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "media/filters/video_renderer_base.h"
      6 
      7 #include "base/bind.h"
      8 #include "base/callback.h"
      9 #include "base/callback_helpers.h"
     10 #include "base/message_loop/message_loop.h"
     11 #include "base/threading/platform_thread.h"
     12 #include "media/base/buffers.h"
     13 #include "media/base/limits.h"
     14 #include "media/base/pipeline.h"
     15 #include "media/base/video_frame.h"
     16 
     17 namespace media {
     18 
     19 base::TimeDelta VideoRendererBase::kMaxLastFrameDuration() {
     20   return base::TimeDelta::FromMilliseconds(250);
     21 }
     22 
     23 VideoRendererBase::VideoRendererBase(
     24     const scoped_refptr<base::MessageLoopProxy>& message_loop,
     25     ScopedVector<VideoDecoder> decoders,
     26     const SetDecryptorReadyCB& set_decryptor_ready_cb,
     27     const PaintCB& paint_cb,
     28     const SetOpaqueCB& set_opaque_cb,
     29     bool drop_frames)
     30     : message_loop_(message_loop),
     31       weak_factory_(this),
     32       video_frame_stream_(
     33           message_loop, decoders.Pass(), set_decryptor_ready_cb),
     34       received_end_of_stream_(false),
     35       frame_available_(&lock_),
     36       state_(kUninitialized),
     37       thread_(),
     38       pending_read_(false),
     39       drop_frames_(drop_frames),
     40       playback_rate_(0),
     41       paint_cb_(paint_cb),
     42       set_opaque_cb_(set_opaque_cb),
     43       last_timestamp_(kNoTimestamp()) {
     44   DCHECK(!paint_cb_.is_null());
     45 }
     46 
     47 VideoRendererBase::~VideoRendererBase() {
     48   base::AutoLock auto_lock(lock_);
     49   CHECK(state_ == kStopped || state_ == kUninitialized) << state_;
     50   CHECK(thread_.is_null());
     51 }
     52 
     53 void VideoRendererBase::Play(const base::Closure& callback) {
     54   DCHECK(message_loop_->BelongsToCurrentThread());
     55   base::AutoLock auto_lock(lock_);
     56   DCHECK_EQ(kPrerolled, state_);
     57   state_ = kPlaying;
     58   callback.Run();
     59 }
     60 
     61 void VideoRendererBase::Pause(const base::Closure& callback) {
     62   DCHECK(message_loop_->BelongsToCurrentThread());
     63   base::AutoLock auto_lock(lock_);
     64   DCHECK(state_ != kUninitialized || state_ == kError);
     65   state_ = kPaused;
     66   callback.Run();
     67 }
     68 
     69 void VideoRendererBase::Flush(const base::Closure& callback) {
     70   DCHECK(message_loop_->BelongsToCurrentThread());
     71   base::AutoLock auto_lock(lock_);
     72   DCHECK_EQ(state_, kPaused);
     73   flush_cb_ = callback;
     74   state_ = kFlushing;
     75 
     76   // This is necessary if the |video_frame_stream_| has already seen an end of
     77   // stream and needs to drain it before flushing it.
     78   ready_frames_.clear();
     79   received_end_of_stream_ = false;
     80   video_frame_stream_.Reset(base::Bind(
     81       &VideoRendererBase::OnVideoFrameStreamResetDone, weak_this_));
     82 }
     83 
     84 void VideoRendererBase::Stop(const base::Closure& callback) {
     85   DCHECK(message_loop_->BelongsToCurrentThread());
     86   base::AutoLock auto_lock(lock_);
     87   if (state_ == kUninitialized || state_ == kStopped) {
     88     callback.Run();
     89     return;
     90   }
     91 
     92   // TODO(scherkus): Consider invalidating |weak_factory_| and replacing
     93   // task-running guards that check |state_| with DCHECK().
     94 
     95   state_ = kStopped;
     96 
     97   statistics_cb_.Reset();
     98   max_time_cb_.Reset();
     99   DoStopOrError_Locked();
    100 
    101   // Clean up our thread if present.
    102   base::PlatformThreadHandle thread_to_join = base::PlatformThreadHandle();
    103   if (!thread_.is_null()) {
    104     // Signal the thread since it's possible to get stopped with the video
    105     // thread waiting for a read to complete.
    106     frame_available_.Signal();
    107     std::swap(thread_, thread_to_join);
    108   }
    109 
    110   if (!thread_to_join.is_null()) {
    111     base::AutoUnlock auto_unlock(lock_);
    112     base::PlatformThread::Join(thread_to_join);
    113   }
    114 
    115   video_frame_stream_.Stop(callback);
    116 }
    117 
    118 void VideoRendererBase::SetPlaybackRate(float playback_rate) {
    119   DCHECK(message_loop_->BelongsToCurrentThread());
    120   base::AutoLock auto_lock(lock_);
    121   playback_rate_ = playback_rate;
    122 }
    123 
    124 void VideoRendererBase::Preroll(base::TimeDelta time,
    125                                 const PipelineStatusCB& cb) {
    126   DCHECK(message_loop_->BelongsToCurrentThread());
    127   base::AutoLock auto_lock(lock_);
    128   DCHECK_EQ(state_, kFlushed) << "Must flush prior to prerolling.";
    129   DCHECK(!cb.is_null());
    130   DCHECK(preroll_cb_.is_null());
    131 
    132   state_ = kPrerolling;
    133   preroll_cb_ = cb;
    134   preroll_timestamp_ = time;
    135   AttemptRead_Locked();
    136 }
    137 
    138 void VideoRendererBase::Initialize(DemuxerStream* stream,
    139                                    const PipelineStatusCB& init_cb,
    140                                    const StatisticsCB& statistics_cb,
    141                                    const TimeCB& max_time_cb,
    142                                    const NaturalSizeChangedCB& size_changed_cb,
    143                                    const base::Closure& ended_cb,
    144                                    const PipelineStatusCB& error_cb,
    145                                    const TimeDeltaCB& get_time_cb,
    146                                    const TimeDeltaCB& get_duration_cb) {
    147   DCHECK(message_loop_->BelongsToCurrentThread());
    148   base::AutoLock auto_lock(lock_);
    149   DCHECK(stream);
    150   DCHECK_EQ(stream->type(), DemuxerStream::VIDEO);
    151   DCHECK(!init_cb.is_null());
    152   DCHECK(!statistics_cb.is_null());
    153   DCHECK(!max_time_cb.is_null());
    154   DCHECK(!size_changed_cb.is_null());
    155   DCHECK(!ended_cb.is_null());
    156   DCHECK(!get_time_cb.is_null());
    157   DCHECK(!get_duration_cb.is_null());
    158   DCHECK_EQ(kUninitialized, state_);
    159 
    160   weak_this_ = weak_factory_.GetWeakPtr();
    161   init_cb_ = init_cb;
    162   statistics_cb_ = statistics_cb;
    163   max_time_cb_ = max_time_cb;
    164   size_changed_cb_ = size_changed_cb;
    165   ended_cb_ = ended_cb;
    166   error_cb_ = error_cb;
    167   get_time_cb_ = get_time_cb;
    168   get_duration_cb_ = get_duration_cb;
    169   state_ = kInitializing;
    170 
    171   video_frame_stream_.Initialize(
    172       stream,
    173       statistics_cb,
    174       base::Bind(&VideoRendererBase::OnVideoFrameStreamInitialized,
    175                  weak_this_));
    176 }
    177 
    178 void VideoRendererBase::OnVideoFrameStreamInitialized(bool success,
    179                                                       bool has_alpha) {
    180   DCHECK(message_loop_->BelongsToCurrentThread());
    181   base::AutoLock auto_lock(lock_);
    182 
    183   if (state_ == kStopped)
    184     return;
    185 
    186   DCHECK_EQ(state_, kInitializing);
    187 
    188   if (!success) {
    189     state_ = kUninitialized;
    190     base::ResetAndReturn(&init_cb_).Run(DECODER_ERROR_NOT_SUPPORTED);
    191     return;
    192   }
    193 
    194   // We're all good!  Consider ourselves flushed. (ThreadMain() should never
    195   // see us in the kUninitialized state).
    196   // Since we had an initial Preroll(), we consider ourself flushed, because we
    197   // have not populated any buffers yet.
    198   state_ = kFlushed;
    199 
    200   set_opaque_cb_.Run(!has_alpha);
    201   set_opaque_cb_.Reset();
    202 
    203   // Create our video thread.
    204   if (!base::PlatformThread::Create(0, this, &thread_)) {
    205     NOTREACHED() << "Video thread creation failed";
    206     state_ = kError;
    207     base::ResetAndReturn(&init_cb_).Run(PIPELINE_ERROR_INITIALIZATION_FAILED);
    208     return;
    209   }
    210 
    211 #if defined(OS_WIN)
    212   // Bump up our priority so our sleeping is more accurate.
    213   // TODO(scherkus): find out if this is necessary, but it seems to help.
    214   ::SetThreadPriority(thread_.platform_handle(), THREAD_PRIORITY_ABOVE_NORMAL);
    215 #endif  // defined(OS_WIN)
    216   base::ResetAndReturn(&init_cb_).Run(PIPELINE_OK);
    217 }
    218 
    219 // PlatformThread::Delegate implementation.
    220 void VideoRendererBase::ThreadMain() {
    221   base::PlatformThread::SetName("CrVideoRenderer");
    222 
    223   // The number of milliseconds to idle when we do not have anything to do.
    224   // Nothing special about the value, other than we're being more OS-friendly
    225   // than sleeping for 1 millisecond.
    226   //
    227   // TODO(scherkus): switch to pure event-driven frame timing instead of this
    228   // kIdleTimeDelta business http://crbug.com/106874
    229   const base::TimeDelta kIdleTimeDelta =
    230       base::TimeDelta::FromMilliseconds(10);
    231 
    232   for (;;) {
    233     base::AutoLock auto_lock(lock_);
    234 
    235     // Thread exit condition.
    236     if (state_ == kStopped)
    237       return;
    238 
    239     // Remain idle as long as we're not playing.
    240     if (state_ != kPlaying || playback_rate_ == 0) {
    241       frame_available_.TimedWait(kIdleTimeDelta);
    242       continue;
    243     }
    244 
    245     // Remain idle until we have the next frame ready for rendering.
    246     if (ready_frames_.empty()) {
    247       if (received_end_of_stream_) {
    248         state_ = kEnded;
    249         ended_cb_.Run();
    250 
    251         // No need to sleep here as we idle when |state_ != kPlaying|.
    252         continue;
    253       }
    254 
    255       frame_available_.TimedWait(kIdleTimeDelta);
    256       continue;
    257     }
    258 
    259     base::TimeDelta remaining_time =
    260         CalculateSleepDuration(ready_frames_.front(), playback_rate_);
    261 
    262     // Sleep up to a maximum of our idle time until we're within the time to
    263     // render the next frame.
    264     if (remaining_time.InMicroseconds() > 0) {
    265       remaining_time = std::min(remaining_time, kIdleTimeDelta);
    266       frame_available_.TimedWait(remaining_time);
    267       continue;
    268     }
    269 
    270     // Deadline is defined as the midpoint between this frame and the next
    271     // frame, using the delta between this frame and the previous frame as the
    272     // assumption for frame duration.
    273     //
    274     // TODO(scherkus): An improvement over midpoint might be selecting the
    275     // minimum and/or maximum between the midpoint and some constants. As a
    276     // thought experiment, consider what would be better than the midpoint
    277     // for both the 1fps case and 120fps case.
    278     //
    279     // TODO(scherkus): This can be vastly improved. Use a histogram to measure
    280     // the accuracy of our frame timing code. http://crbug.com/149829
    281     if (drop_frames_ && last_timestamp_ != kNoTimestamp()) {
    282       base::TimeDelta now = get_time_cb_.Run();
    283       base::TimeDelta deadline = ready_frames_.front()->GetTimestamp() +
    284           (ready_frames_.front()->GetTimestamp() - last_timestamp_) / 2;
    285 
    286       if (now > deadline) {
    287         DropNextReadyFrame_Locked();
    288         continue;
    289       }
    290     }
    291 
    292     // Congratulations! You've made it past the video frame timing gauntlet.
    293     //
    294     // At this point enough time has passed that the next frame that ready for
    295     // rendering.
    296     PaintNextReadyFrame_Locked();
    297   }
    298 }
    299 
    300 void VideoRendererBase::PaintNextReadyFrame_Locked() {
    301   lock_.AssertAcquired();
    302 
    303   scoped_refptr<VideoFrame> next_frame = ready_frames_.front();
    304   ready_frames_.pop_front();
    305 
    306   last_timestamp_ = next_frame->GetTimestamp();
    307 
    308   const gfx::Size& natural_size = next_frame->natural_size();
    309   if (natural_size != last_natural_size_) {
    310     last_natural_size_ = natural_size;
    311     size_changed_cb_.Run(natural_size);
    312   }
    313 
    314   paint_cb_.Run(next_frame);
    315 
    316   message_loop_->PostTask(FROM_HERE, base::Bind(
    317       &VideoRendererBase::AttemptRead, weak_this_));
    318 }
    319 
    320 void VideoRendererBase::DropNextReadyFrame_Locked() {
    321   lock_.AssertAcquired();
    322 
    323   last_timestamp_ = ready_frames_.front()->GetTimestamp();
    324   ready_frames_.pop_front();
    325 
    326   PipelineStatistics statistics;
    327   statistics.video_frames_dropped = 1;
    328   statistics_cb_.Run(statistics);
    329 
    330   message_loop_->PostTask(FROM_HERE, base::Bind(
    331       &VideoRendererBase::AttemptRead, weak_this_));
    332 }
    333 
    334 void VideoRendererBase::FrameReady(VideoFrameStream::Status status,
    335                                    const scoped_refptr<VideoFrame>& frame) {
    336   base::AutoLock auto_lock(lock_);
    337   DCHECK_NE(state_, kUninitialized);
    338   DCHECK_NE(state_, kFlushed);
    339 
    340   CHECK(pending_read_);
    341   pending_read_ = false;
    342 
    343   if (status == VideoFrameStream::DECODE_ERROR ||
    344       status == VideoFrameStream::DECRYPT_ERROR) {
    345     DCHECK(!frame.get());
    346     PipelineStatus error = PIPELINE_ERROR_DECODE;
    347     if (status == VideoFrameStream::DECRYPT_ERROR)
    348       error = PIPELINE_ERROR_DECRYPT;
    349 
    350     if (!preroll_cb_.is_null()) {
    351       base::ResetAndReturn(&preroll_cb_).Run(error);
    352       return;
    353     }
    354 
    355     error_cb_.Run(error);
    356     return;
    357   }
    358 
    359   // Already-queued VideoFrameStream ReadCB's can fire after various state
    360   // transitions have happened; in that case just drop those frames immediately.
    361   if (state_ == kStopped || state_ == kError || state_ == kFlushing)
    362     return;
    363 
    364   if (!frame.get()) {
    365     // Abort preroll early for a NULL frame because we won't get more frames.
    366     // A new preroll will be requested after this one completes so there is no
    367     // point trying to collect more frames.
    368     if (state_ == kPrerolling)
    369       TransitionToPrerolled_Locked();
    370 
    371     return;
    372   }
    373 
    374   if (frame->IsEndOfStream()) {
    375     DCHECK(!received_end_of_stream_);
    376     received_end_of_stream_ = true;
    377     max_time_cb_.Run(get_duration_cb_.Run());
    378 
    379     if (state_ == kPrerolling)
    380       TransitionToPrerolled_Locked();
    381 
    382     return;
    383   }
    384 
    385   // Maintain the latest frame decoded so the correct frame is displayed after
    386   // prerolling has completed.
    387   if (state_ == kPrerolling && frame->GetTimestamp() <= preroll_timestamp_) {
    388     ready_frames_.clear();
    389   }
    390 
    391   AddReadyFrame_Locked(frame);
    392 
    393   if (state_ == kPrerolling) {
    394     if (!video_frame_stream_.CanReadWithoutStalling() ||
    395         ready_frames_.size() >= static_cast<size_t>(limits::kMaxVideoFrames)) {
    396       TransitionToPrerolled_Locked();
    397     }
    398   } else {
    399     // We only count frames decoded during normal playback.
    400     PipelineStatistics statistics;
    401     statistics.video_frames_decoded = 1;
    402     statistics_cb_.Run(statistics);
    403   }
    404 
    405   // Always request more decoded video if we have capacity. This serves two
    406   // purposes:
    407   //   1) Prerolling while paused
    408   //   2) Keeps decoding going if video rendering thread starts falling behind
    409   AttemptRead_Locked();
    410 }
    411 
    412 void VideoRendererBase::AddReadyFrame_Locked(
    413     const scoped_refptr<VideoFrame>& frame) {
    414   lock_.AssertAcquired();
    415   DCHECK(!frame->IsEndOfStream());
    416 
    417   // Adjust the incoming frame if its rendering stop time is past the duration
    418   // of the video itself. This is typically the last frame of the video and
    419   // occurs if the container specifies a duration that isn't a multiple of the
    420   // frame rate.  Another way for this to happen is for the container to state
    421   // a smaller duration than the largest packet timestamp.
    422   base::TimeDelta duration = get_duration_cb_.Run();
    423   if (frame->GetTimestamp() > duration) {
    424     frame->SetTimestamp(duration);
    425   }
    426 
    427   ready_frames_.push_back(frame);
    428   DCHECK_LE(ready_frames_.size(),
    429             static_cast<size_t>(limits::kMaxVideoFrames));
    430 
    431   max_time_cb_.Run(frame->GetTimestamp());
    432 
    433   // Avoid needlessly waking up |thread_| unless playing.
    434   if (state_ == kPlaying)
    435     frame_available_.Signal();
    436 }
    437 
    438 void VideoRendererBase::AttemptRead() {
    439   base::AutoLock auto_lock(lock_);
    440   AttemptRead_Locked();
    441 }
    442 
    443 void VideoRendererBase::AttemptRead_Locked() {
    444   DCHECK(message_loop_->BelongsToCurrentThread());
    445   lock_.AssertAcquired();
    446 
    447   if (pending_read_ || received_end_of_stream_ ||
    448       ready_frames_.size() == static_cast<size_t>(limits::kMaxVideoFrames)) {
    449     return;
    450   }
    451 
    452   switch (state_) {
    453     case kPaused:
    454     case kPrerolling:
    455     case kPlaying:
    456       pending_read_ = true;
    457       video_frame_stream_.Read(base::Bind(&VideoRendererBase::FrameReady,
    458                                           weak_this_));
    459       return;
    460 
    461     case kUninitialized:
    462     case kInitializing:
    463     case kPrerolled:
    464     case kFlushing:
    465     case kFlushed:
    466     case kEnded:
    467     case kStopped:
    468     case kError:
    469       return;
    470   }
    471 }
    472 
    473 void VideoRendererBase::OnVideoFrameStreamResetDone() {
    474   base::AutoLock auto_lock(lock_);
    475   if (state_ == kStopped)
    476     return;
    477 
    478   DCHECK_EQ(kFlushing, state_);
    479   DCHECK(!pending_read_);
    480   DCHECK(ready_frames_.empty());
    481   DCHECK(!received_end_of_stream_);
    482 
    483   state_ = kFlushed;
    484   last_timestamp_ = kNoTimestamp();
    485   base::ResetAndReturn(&flush_cb_).Run();
    486 }
    487 
    488 base::TimeDelta VideoRendererBase::CalculateSleepDuration(
    489     const scoped_refptr<VideoFrame>& next_frame,
    490     float playback_rate) {
    491   // Determine the current and next presentation timestamps.
    492   base::TimeDelta now = get_time_cb_.Run();
    493   base::TimeDelta next_pts = next_frame->GetTimestamp();
    494 
    495   // Scale our sleep based on the playback rate.
    496   base::TimeDelta sleep = next_pts - now;
    497   return base::TimeDelta::FromMicroseconds(
    498       static_cast<int64>(sleep.InMicroseconds() / playback_rate));
    499 }
    500 
    501 void VideoRendererBase::DoStopOrError_Locked() {
    502   lock_.AssertAcquired();
    503   last_timestamp_ = kNoTimestamp();
    504   ready_frames_.clear();
    505 }
    506 
    507 void VideoRendererBase::TransitionToPrerolled_Locked() {
    508   lock_.AssertAcquired();
    509   DCHECK_EQ(state_, kPrerolling);
    510 
    511   state_ = kPrerolled;
    512 
    513   // Because we might remain in the prerolled state for an undetermined amount
    514   // of time (e.g., we seeked while paused), we'll paint the first prerolled
    515   // frame.
    516   if (!ready_frames_.empty())
    517     PaintNextReadyFrame_Locked();
    518 
    519   base::ResetAndReturn(&preroll_cb_).Run(PIPELINE_OK);
    520 }
    521 
    522 }  // namespace media
    523