Home | History | Annotate | Download | only in filters 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "media/filters/audio_renderer_impl.h"
      6 
      7 #include <math.h>
      8 
      9 #include <algorithm>
     10 
     11 #include "base/bind.h"
     12 #include "base/callback.h"
     13 #include "base/callback_helpers.h"
     14 #include "base/logging.h"
     15 #include "base/message_loop/message_loop_proxy.h"
     16 #include "base/metrics/histogram.h"
     17 #include "media/audio/audio_util.h"
     18 #include "media/base/audio_buffer.h"
     19 #include "media/base/audio_splicer.h"
     20 #include "media/base/bind_to_loop.h"
     21 #include "media/base/demuxer_stream.h"
     22 #include "media/filters/audio_decoder_selector.h"
     23 #include "media/filters/decrypting_demuxer_stream.h"
     24 
     25 namespace media {
     26 
     27 namespace {
     28 
     29 enum AudioRendererEvent {
     30   INITIALIZED,
     31   RENDER_ERROR,
     32   MAX_EVENTS
     33 };
     34 
     35 void HistogramRendererEvent(AudioRendererEvent event) {
     36   UMA_HISTOGRAM_ENUMERATION("Media.AudioRendererEvents", event, MAX_EVENTS);
     37 }
     38 
     39 }  // namespace
     40 
     41 AudioRendererImpl::AudioRendererImpl(
     42     const scoped_refptr<base::MessageLoopProxy>& message_loop,
     43     media::AudioRendererSink* sink,
     44     ScopedVector<AudioDecoder> decoders,
     45     const SetDecryptorReadyCB& set_decryptor_ready_cb,
     46     bool increase_preroll_on_underflow)
     47     : message_loop_(message_loop),
     48       weak_factory_(this),
     49       sink_(sink),
     50       decoder_selector_(new AudioDecoderSelector(
     51           message_loop, decoders.Pass(), set_decryptor_ready_cb)),
     52       now_cb_(base::Bind(&base::TimeTicks::Now)),
     53       state_(kUninitialized),
     54       sink_playing_(false),
     55       pending_read_(false),
     56       received_end_of_stream_(false),
     57       rendered_end_of_stream_(false),
     58       audio_time_buffered_(kNoTimestamp()),
     59       current_time_(kNoTimestamp()),
     60       underflow_disabled_(false),
     61       increase_preroll_on_underflow_(increase_preroll_on_underflow),
     62       preroll_aborted_(false) {
     63 }
     64 
     65 AudioRendererImpl::~AudioRendererImpl() {
     66   // Stop() should have been called and |algorithm_| should have been destroyed.
     67   DCHECK(state_ == kUninitialized || state_ == kStopped);
     68   DCHECK(!algorithm_.get());
     69 }
     70 
     71 void AudioRendererImpl::Play(const base::Closure& callback) {
     72   DCHECK(message_loop_->BelongsToCurrentThread());
     73 
     74   {
     75     base::AutoLock auto_lock(lock_);
     76     DCHECK_EQ(state_, kPaused);
     77     state_ = kPlaying;
     78     callback.Run();
     79     earliest_end_time_ = now_cb_.Run();
     80   }
     81 
     82   if (algorithm_->playback_rate() != 0)
     83     DoPlay();
     84   else
     85     DCHECK(!sink_playing_);
     86 }
     87 
     88 void AudioRendererImpl::DoPlay() {
     89   DCHECK(message_loop_->BelongsToCurrentThread());
     90   {
     91     base::AutoLock auto_lock(lock_);
     92     earliest_end_time_ = now_cb_.Run();
     93   }
     94 
     95   if (state_ == kPlaying && !sink_playing_) {
     96     sink_->Play();
     97     sink_playing_ = true;
     98   }
     99 }
    100 
    101 void AudioRendererImpl::Pause(const base::Closure& callback) {
    102   DCHECK(message_loop_->BelongsToCurrentThread());
    103 
    104   {
    105     base::AutoLock auto_lock(lock_);
    106     DCHECK(state_ == kPlaying || state_ == kUnderflow ||
    107            state_ == kRebuffering) << "state_ == " << state_;
    108     pause_cb_ = callback;
    109     state_ = kPaused;
    110 
    111     // Pause only when we've completed our pending read.
    112     if (!pending_read_)
    113       base::ResetAndReturn(&pause_cb_).Run();
    114   }
    115 
    116   DoPause();
    117 }
    118 
    119 void AudioRendererImpl::DoPause() {
    120   DCHECK(message_loop_->BelongsToCurrentThread());
    121   if (sink_playing_) {
    122     sink_->Pause();
    123     sink_playing_ = false;
    124   }
    125 }
    126 
    127 void AudioRendererImpl::Flush(const base::Closure& callback) {
    128   DCHECK(message_loop_->BelongsToCurrentThread());
    129 
    130   if (decrypting_demuxer_stream_) {
    131     decrypting_demuxer_stream_->Reset(base::Bind(
    132         &AudioRendererImpl::ResetDecoder, weak_this_, callback));
    133     return;
    134   }
    135 
    136   decoder_->Reset(callback);
    137 }
    138 
    139 void AudioRendererImpl::ResetDecoder(const base::Closure& callback) {
    140   DCHECK(message_loop_->BelongsToCurrentThread());
    141   decoder_->Reset(callback);
    142 }
    143 
    144 void AudioRendererImpl::Stop(const base::Closure& callback) {
    145   DCHECK(message_loop_->BelongsToCurrentThread());
    146   DCHECK(!callback.is_null());
    147 
    148   // TODO(scherkus): Consider invalidating |weak_factory_| and replacing
    149   // task-running guards that check |state_| with DCHECK().
    150 
    151   if (sink_.get()) {
    152     sink_->Stop();
    153     sink_ = NULL;
    154   }
    155 
    156   {
    157     base::AutoLock auto_lock(lock_);
    158     state_ = kStopped;
    159     algorithm_.reset(NULL);
    160     init_cb_.Reset();
    161     underflow_cb_.Reset();
    162     time_cb_.Reset();
    163   }
    164 
    165   callback.Run();
    166 }
    167 
    168 void AudioRendererImpl::Preroll(base::TimeDelta time,
    169                                 const PipelineStatusCB& cb) {
    170   DCHECK(message_loop_->BelongsToCurrentThread());
    171 
    172   base::AutoLock auto_lock(lock_);
    173   DCHECK(!sink_playing_);
    174   DCHECK_EQ(state_, kPaused);
    175   DCHECK(!pending_read_) << "Pending read must complete before seeking";
    176   DCHECK(pause_cb_.is_null());
    177   DCHECK(preroll_cb_.is_null());
    178 
    179   state_ = kPrerolling;
    180   preroll_cb_ = cb;
    181   preroll_timestamp_ = time;
    182 
    183   // Throw away everything and schedule our reads.
    184   audio_time_buffered_ = kNoTimestamp();
    185   current_time_ = kNoTimestamp();
    186   received_end_of_stream_ = false;
    187   rendered_end_of_stream_ = false;
    188   preroll_aborted_ = false;
    189 
    190   splicer_->Reset();
    191   algorithm_->FlushBuffers();
    192   earliest_end_time_ = now_cb_.Run();
    193 
    194   AttemptRead_Locked();
    195 }
    196 
    197 void AudioRendererImpl::Initialize(DemuxerStream* stream,
    198                                    const PipelineStatusCB& init_cb,
    199                                    const StatisticsCB& statistics_cb,
    200                                    const base::Closure& underflow_cb,
    201                                    const TimeCB& time_cb,
    202                                    const base::Closure& ended_cb,
    203                                    const base::Closure& disabled_cb,
    204                                    const PipelineStatusCB& error_cb) {
    205   DCHECK(message_loop_->BelongsToCurrentThread());
    206   DCHECK(stream);
    207   DCHECK_EQ(stream->type(), DemuxerStream::AUDIO);
    208   DCHECK(!init_cb.is_null());
    209   DCHECK(!statistics_cb.is_null());
    210   DCHECK(!underflow_cb.is_null());
    211   DCHECK(!time_cb.is_null());
    212   DCHECK(!ended_cb.is_null());
    213   DCHECK(!disabled_cb.is_null());
    214   DCHECK(!error_cb.is_null());
    215   DCHECK_EQ(kUninitialized, state_);
    216   DCHECK(sink_.get());
    217 
    218   weak_this_ = weak_factory_.GetWeakPtr();
    219   init_cb_ = init_cb;
    220   statistics_cb_ = statistics_cb;
    221   underflow_cb_ = underflow_cb;
    222   time_cb_ = time_cb;
    223   ended_cb_ = ended_cb;
    224   disabled_cb_ = disabled_cb;
    225   error_cb_ = error_cb;
    226 
    227   decoder_selector_->SelectAudioDecoder(
    228       stream,
    229       statistics_cb,
    230       base::Bind(&AudioRendererImpl::OnDecoderSelected, weak_this_));
    231 }
    232 
    233 void AudioRendererImpl::OnDecoderSelected(
    234     scoped_ptr<AudioDecoder> decoder,
    235     scoped_ptr<DecryptingDemuxerStream> decrypting_demuxer_stream) {
    236   DCHECK(message_loop_->BelongsToCurrentThread());
    237   scoped_ptr<AudioDecoderSelector> deleter(decoder_selector_.Pass());
    238 
    239   if (state_ == kStopped) {
    240     DCHECK(!sink_.get());
    241     return;
    242   }
    243 
    244   if (!decoder) {
    245     base::ResetAndReturn(&init_cb_).Run(DECODER_ERROR_NOT_SUPPORTED);
    246     return;
    247   }
    248 
    249   decoder_ = decoder.Pass();
    250   decrypting_demuxer_stream_ = decrypting_demuxer_stream.Pass();
    251 
    252   int sample_rate = decoder_->samples_per_second();
    253 
    254   // The actual buffer size is controlled via the size of the AudioBus provided
    255   // to Render(), so just choose something reasonable here for looks.
    256   int buffer_size = decoder_->samples_per_second() / 100;
    257   audio_parameters_ = AudioParameters(
    258       AudioParameters::AUDIO_PCM_LOW_LATENCY, decoder_->channel_layout(),
    259       sample_rate, decoder_->bits_per_channel(), buffer_size);
    260   if (!audio_parameters_.IsValid()) {
    261     base::ResetAndReturn(&init_cb_).Run(PIPELINE_ERROR_INITIALIZATION_FAILED);
    262     return;
    263   }
    264 
    265   splicer_.reset(new AudioSplicer(sample_rate));
    266 
    267   // We're all good! Continue initializing the rest of the audio renderer based
    268   // on the decoder format.
    269   algorithm_.reset(new AudioRendererAlgorithm());
    270   algorithm_->Initialize(0, audio_parameters_);
    271 
    272   state_ = kPaused;
    273 
    274   HistogramRendererEvent(INITIALIZED);
    275 
    276   sink_->Initialize(audio_parameters_, weak_this_.get());
    277   sink_->Start();
    278 
    279   // Some sinks play on start...
    280   sink_->Pause();
    281   DCHECK(!sink_playing_);
    282 
    283   base::ResetAndReturn(&init_cb_).Run(PIPELINE_OK);
    284 }
    285 
    286 void AudioRendererImpl::ResumeAfterUnderflow() {
    287   DCHECK(message_loop_->BelongsToCurrentThread());
    288   base::AutoLock auto_lock(lock_);
    289   if (state_ == kUnderflow) {
    290     // The "&& preroll_aborted_" is a hack. If preroll is aborted, then we
    291     // shouldn't even reach the kUnderflow state to begin with. But for now
    292     // we're just making sure that the audio buffer capacity (i.e. the
    293     // number of bytes that need to be buffered for preroll to complete)
    294     // does not increase due to an aborted preroll.
    295     // TODO(vrk): Fix this bug correctly! (crbug.com/151352)
    296     if (increase_preroll_on_underflow_ && !preroll_aborted_)
    297       algorithm_->IncreaseQueueCapacity();
    298 
    299     state_ = kRebuffering;
    300   }
    301 }
    302 
    303 void AudioRendererImpl::SetVolume(float volume) {
    304   DCHECK(message_loop_->BelongsToCurrentThread());
    305   DCHECK(sink_.get());
    306   sink_->SetVolume(volume);
    307 }
    308 
    309 void AudioRendererImpl::DecodedAudioReady(
    310     AudioDecoder::Status status,
    311     const scoped_refptr<AudioBuffer>& buffer) {
    312   DCHECK(message_loop_->BelongsToCurrentThread());
    313 
    314   base::AutoLock auto_lock(lock_);
    315   DCHECK(state_ == kPaused || state_ == kPrerolling || state_ == kPlaying ||
    316          state_ == kUnderflow || state_ == kRebuffering || state_ == kStopped);
    317 
    318   CHECK(pending_read_);
    319   pending_read_ = false;
    320 
    321   if (status == AudioDecoder::kAborted) {
    322     HandleAbortedReadOrDecodeError(false);
    323     return;
    324   }
    325 
    326   if (status == AudioDecoder::kDecodeError) {
    327     HandleAbortedReadOrDecodeError(true);
    328     return;
    329   }
    330 
    331   DCHECK_EQ(status, AudioDecoder::kOk);
    332   DCHECK(buffer.get());
    333 
    334   if (!splicer_->AddInput(buffer)) {
    335     HandleAbortedReadOrDecodeError(true);
    336     return;
    337   }
    338 
    339   if (!splicer_->HasNextBuffer()) {
    340     AttemptRead_Locked();
    341     return;
    342   }
    343 
    344   bool need_another_buffer = false;
    345   while (splicer_->HasNextBuffer())
    346     need_another_buffer = HandleSplicerBuffer(splicer_->GetNextBuffer());
    347 
    348   if (!need_another_buffer && !CanRead_Locked())
    349     return;
    350 
    351   AttemptRead_Locked();
    352 }
    353 
    354 bool AudioRendererImpl::HandleSplicerBuffer(
    355     const scoped_refptr<AudioBuffer>& buffer) {
    356   if (buffer->end_of_stream()) {
    357     received_end_of_stream_ = true;
    358 
    359     // Transition to kPlaying if we are currently handling an underflow since
    360     // no more data will be arriving.
    361     if (state_ == kUnderflow || state_ == kRebuffering)
    362       state_ = kPlaying;
    363   }
    364 
    365   switch (state_) {
    366     case kUninitialized:
    367       NOTREACHED();
    368       return false;
    369     case kPaused:
    370       if (!buffer->end_of_stream())
    371         algorithm_->EnqueueBuffer(buffer);
    372       DCHECK(!pending_read_);
    373       base::ResetAndReturn(&pause_cb_).Run();
    374       return false;
    375     case kPrerolling:
    376       if (IsBeforePrerollTime(buffer))
    377         return true;
    378 
    379       if (!buffer->end_of_stream()) {
    380         algorithm_->EnqueueBuffer(buffer);
    381         if (!algorithm_->IsQueueFull())
    382           return false;
    383       }
    384       state_ = kPaused;
    385       base::ResetAndReturn(&preroll_cb_).Run(PIPELINE_OK);
    386       return false;
    387     case kPlaying:
    388     case kUnderflow:
    389     case kRebuffering:
    390       if (!buffer->end_of_stream())
    391         algorithm_->EnqueueBuffer(buffer);
    392       return false;
    393     case kStopped:
    394       return false;
    395   }
    396   return false;
    397 }
    398 
    399 void AudioRendererImpl::AttemptRead() {
    400   base::AutoLock auto_lock(lock_);
    401   AttemptRead_Locked();
    402 }
    403 
    404 void AudioRendererImpl::AttemptRead_Locked() {
    405   DCHECK(message_loop_->BelongsToCurrentThread());
    406   lock_.AssertAcquired();
    407 
    408   if (!CanRead_Locked())
    409     return;
    410 
    411   pending_read_ = true;
    412   decoder_->Read(base::Bind(&AudioRendererImpl::DecodedAudioReady, weak_this_));
    413 }
    414 
    415 bool AudioRendererImpl::CanRead_Locked() {
    416   lock_.AssertAcquired();
    417 
    418   switch (state_) {
    419     case kUninitialized:
    420     case kPaused:
    421     case kStopped:
    422       return false;
    423 
    424     case kPrerolling:
    425     case kPlaying:
    426     case kUnderflow:
    427     case kRebuffering:
    428       break;
    429   }
    430 
    431   return !pending_read_ && !received_end_of_stream_ &&
    432       !algorithm_->IsQueueFull();
    433 }
    434 
    435 void AudioRendererImpl::SetPlaybackRate(float playback_rate) {
    436   DCHECK(message_loop_->BelongsToCurrentThread());
    437   DCHECK_GE(playback_rate, 0);
    438   DCHECK(sink_.get());
    439 
    440   // We have two cases here:
    441   // Play: current_playback_rate == 0 && playback_rate != 0
    442   // Pause: current_playback_rate != 0 && playback_rate == 0
    443   float current_playback_rate = algorithm_->playback_rate();
    444   if (current_playback_rate == 0 && playback_rate != 0)
    445     DoPlay();
    446   else if (current_playback_rate != 0 && playback_rate == 0)
    447     DoPause();
    448 
    449   base::AutoLock auto_lock(lock_);
    450   algorithm_->SetPlaybackRate(playback_rate);
    451 }
    452 
    453 bool AudioRendererImpl::IsBeforePrerollTime(
    454     const scoped_refptr<AudioBuffer>& buffer) {
    455   return (state_ == kPrerolling) && buffer.get() && !buffer->end_of_stream() &&
    456          (buffer->timestamp() + buffer->duration()) < preroll_timestamp_;
    457 }
    458 
    459 int AudioRendererImpl::Render(AudioBus* audio_bus,
    460                               int audio_delay_milliseconds) {
    461   int frames_filled =
    462       FillBuffer(audio_bus, audio_bus->frames(), audio_delay_milliseconds);
    463   DCHECK_LE(frames_filled, audio_bus->frames());
    464   return frames_filled;
    465 }
    466 
    467 uint32 AudioRendererImpl::FillBuffer(AudioBus* dest,
    468                                      uint32 requested_frames,
    469                                      int audio_delay_milliseconds) {
    470   base::TimeDelta current_time = kNoTimestamp();
    471   base::TimeDelta max_time = kNoTimestamp();
    472   base::TimeDelta playback_delay = base::TimeDelta::FromMilliseconds(
    473       audio_delay_milliseconds);
    474 
    475   size_t frames_written = 0;
    476   base::Closure underflow_cb;
    477   {
    478     base::AutoLock auto_lock(lock_);
    479 
    480     // Ensure Stop() hasn't destroyed our |algorithm_| on the pipeline thread.
    481     if (!algorithm_)
    482       return 0;
    483 
    484     float playback_rate = algorithm_->playback_rate();
    485     if (playback_rate == 0)
    486       return 0;
    487 
    488     if (state_ == kRebuffering && algorithm_->IsQueueFull())
    489       state_ = kPlaying;
    490 
    491     // Mute audio by returning 0 when not playing.
    492     if (state_ != kPlaying)
    493       return 0;
    494 
    495     // We use the following conditions to determine end of playback:
    496     //   1) Algorithm can not fill the audio callback buffer
    497     //   2) We received an end of stream buffer
    498     //   3) We haven't already signalled that we've ended
    499     //   4) Our estimated earliest end time has expired
    500     //
    501     // TODO(enal): we should replace (4) with a check that the browser has no
    502     // more audio data or at least use a delayed callback.
    503     //
    504     // We use the following conditions to determine underflow:
    505     //   1) Algorithm can not fill the audio callback buffer
    506     //   2) We have NOT received an end of stream buffer
    507     //   3) We are in the kPlaying state
    508     //
    509     // Otherwise the buffer has data we can send to the device.
    510     frames_written = algorithm_->FillBuffer(dest, requested_frames);
    511     if (frames_written == 0) {
    512       const base::TimeTicks now = now_cb_.Run();
    513 
    514       if (received_end_of_stream_ && !rendered_end_of_stream_ &&
    515           now >= earliest_end_time_) {
    516         rendered_end_of_stream_ = true;
    517         ended_cb_.Run();
    518       } else if (!received_end_of_stream_ && state_ == kPlaying &&
    519                  !underflow_disabled_) {
    520         state_ = kUnderflow;
    521         underflow_cb = underflow_cb_;
    522       } else {
    523         // We can't write any data this cycle. For example, we may have
    524         // sent all available data to the audio device while not reaching
    525         // |earliest_end_time_|.
    526       }
    527     }
    528 
    529     if (CanRead_Locked()) {
    530       message_loop_->PostTask(FROM_HERE, base::Bind(
    531           &AudioRendererImpl::AttemptRead, weak_this_));
    532     }
    533 
    534     // The |audio_time_buffered_| is the ending timestamp of the last frame
    535     // buffered at the audio device. |playback_delay| is the amount of time
    536     // buffered at the audio device. The current time can be computed by their
    537     // difference.
    538     if (audio_time_buffered_ != kNoTimestamp()) {
    539       // Adjust the delay according to playback rate.
    540       base::TimeDelta adjusted_playback_delay =
    541           base::TimeDelta::FromMicroseconds(ceil(
    542               playback_delay.InMicroseconds() * playback_rate));
    543 
    544       base::TimeDelta previous_time = current_time_;
    545       current_time_ = audio_time_buffered_ - adjusted_playback_delay;
    546 
    547       // Time can change in one of two ways:
    548       //   1) The time of the audio data at the audio device changed, or
    549       //   2) The playback delay value has changed
    550       //
    551       // We only want to set |current_time| (and thus execute |time_cb_|) if
    552       // time has progressed and we haven't signaled end of stream yet.
    553       //
    554       // Why? The current latency of the system results in getting the last call
    555       // to FillBuffer() later than we'd like, which delays firing the 'ended'
    556       // event, which delays the looping/trigging performance of short sound
    557       // effects.
    558       //
    559       // TODO(scherkus): revisit this and switch back to relying on playback
    560       // delay after we've revamped our audio IPC subsystem.
    561       if (current_time_ > previous_time && !rendered_end_of_stream_) {
    562         current_time = current_time_;
    563       }
    564     }
    565 
    566     // The call to FillBuffer() on |algorithm_| has increased the amount of
    567     // buffered audio data. Update the new amount of time buffered.
    568     max_time = algorithm_->GetTime();
    569     audio_time_buffered_ = max_time;
    570 
    571     UpdateEarliestEndTime_Locked(
    572         frames_written, playback_delay, now_cb_.Run());
    573   }
    574 
    575   if (current_time != kNoTimestamp() && max_time != kNoTimestamp()) {
    576     time_cb_.Run(current_time, max_time);
    577   }
    578 
    579   if (!underflow_cb.is_null())
    580     underflow_cb.Run();
    581 
    582   return frames_written;
    583 }
    584 
    585 void AudioRendererImpl::UpdateEarliestEndTime_Locked(
    586     int frames_filled, const base::TimeDelta& playback_delay,
    587     const base::TimeTicks& time_now) {
    588   if (frames_filled <= 0)
    589     return;
    590 
    591   base::TimeDelta predicted_play_time = base::TimeDelta::FromMicroseconds(
    592       static_cast<float>(frames_filled) * base::Time::kMicrosecondsPerSecond /
    593       audio_parameters_.sample_rate());
    594 
    595   lock_.AssertAcquired();
    596   earliest_end_time_ = std::max(
    597       earliest_end_time_, time_now + playback_delay + predicted_play_time);
    598 }
    599 
    600 void AudioRendererImpl::OnRenderError() {
    601   HistogramRendererEvent(RENDER_ERROR);
    602   disabled_cb_.Run();
    603 }
    604 
    605 void AudioRendererImpl::DisableUnderflowForTesting() {
    606   underflow_disabled_ = true;
    607 }
    608 
    609 void AudioRendererImpl::HandleAbortedReadOrDecodeError(bool is_decode_error) {
    610   PipelineStatus status = is_decode_error ? PIPELINE_ERROR_DECODE : PIPELINE_OK;
    611   switch (state_) {
    612     case kUninitialized:
    613       NOTREACHED();
    614       return;
    615     case kPaused:
    616       if (status != PIPELINE_OK)
    617         error_cb_.Run(status);
    618       base::ResetAndReturn(&pause_cb_).Run();
    619       return;
    620     case kPrerolling:
    621       // This is a signal for abort if it's not an error.
    622       preroll_aborted_ = !is_decode_error;
    623       state_ = kPaused;
    624       base::ResetAndReturn(&preroll_cb_).Run(status);
    625       return;
    626     case kPlaying:
    627     case kUnderflow:
    628     case kRebuffering:
    629     case kStopped:
    630       if (status != PIPELINE_OK)
    631         error_cb_.Run(status);
    632       return;
    633   }
    634 }
    635 
    636 }  // namespace media
    637