Home | History | Annotate | Download | only in media 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "content/renderer/media/webrtc_audio_renderer.h"
      6 
      7 #include "base/logging.h"
      8 #include "base/metrics/histogram.h"
      9 #include "base/strings/string_util.h"
     10 #include "base/strings/stringprintf.h"
     11 #include "content/renderer/media/audio_device_factory.h"
     12 #include "content/renderer/media/media_stream_dispatcher.h"
     13 #include "content/renderer/media/webrtc_audio_device_impl.h"
     14 #include "content/renderer/media/webrtc_logging.h"
     15 #include "content/renderer/render_frame_impl.h"
     16 #include "media/audio/audio_output_device.h"
     17 #include "media/audio/audio_parameters.h"
     18 #include "media/audio/sample_rates.h"
     19 #include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h"
     20 #include "third_party/libjingle/source/talk/media/base/audiorenderer.h"
     21 
     22 
     23 #if defined(OS_WIN)
     24 #include "base/win/windows_version.h"
     25 #include "media/audio/win/core_audio_util_win.h"
     26 #endif
     27 
     28 namespace content {
     29 
     30 namespace {
     31 
     32 // We add a UMA histogram measuring the execution time of the Render() method
     33 // every |kNumCallbacksBetweenRenderTimeHistograms| callback. Assuming 10ms
     34 // between each callback leads to one UMA update each 100ms.
     35 const int kNumCallbacksBetweenRenderTimeHistograms = 10;
     36 
     37 // This is a simple wrapper class that's handed out to users of a shared
     38 // WebRtcAudioRenderer instance.  This class maintains the per-user 'playing'
     39 // and 'started' states to avoid problems related to incorrect usage which
     40 // might violate the implementation assumptions inside WebRtcAudioRenderer
     41 // (see the play reference count).
     42 class SharedAudioRenderer : public MediaStreamAudioRenderer {
     43  public:
     44   // Callback definition for a callback that is called when when Play(), Pause()
     45   // or SetVolume are called (whenever the internal |playing_state_| changes).
     46   typedef base::Callback<
     47       void(const scoped_refptr<webrtc::MediaStreamInterface>&,
     48            WebRtcAudioRenderer::PlayingState*)> OnPlayStateChanged;
     49 
     50   SharedAudioRenderer(
     51       const scoped_refptr<MediaStreamAudioRenderer>& delegate,
     52       const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
     53       const OnPlayStateChanged& on_play_state_changed)
     54       : delegate_(delegate), media_stream_(media_stream), started_(false),
     55         on_play_state_changed_(on_play_state_changed) {
     56     DCHECK(!on_play_state_changed_.is_null());
     57     DCHECK(media_stream_.get());
     58   }
     59 
     60  protected:
     61   virtual ~SharedAudioRenderer() {
     62     DCHECK(thread_checker_.CalledOnValidThread());
     63     DVLOG(1) << __FUNCTION__;
     64     Stop();
     65   }
     66 
     67   virtual void Start() OVERRIDE {
     68     DCHECK(thread_checker_.CalledOnValidThread());
     69     if (started_)
     70       return;
     71     started_ = true;
     72     delegate_->Start();
     73   }
     74 
     75   virtual void Play() OVERRIDE {
     76     DCHECK(thread_checker_.CalledOnValidThread());
     77     DCHECK(started_);
     78     if (playing_state_.playing())
     79       return;
     80     playing_state_.set_playing(true);
     81     on_play_state_changed_.Run(media_stream_, &playing_state_);
     82   }
     83 
     84   virtual void Pause() OVERRIDE {
     85     DCHECK(thread_checker_.CalledOnValidThread());
     86     DCHECK(started_);
     87     if (!playing_state_.playing())
     88       return;
     89     playing_state_.set_playing(false);
     90     on_play_state_changed_.Run(media_stream_, &playing_state_);
     91   }
     92 
     93   virtual void Stop() OVERRIDE {
     94     DCHECK(thread_checker_.CalledOnValidThread());
     95     if (!started_)
     96       return;
     97     Pause();
     98     started_ = false;
     99     delegate_->Stop();
    100   }
    101 
    102   virtual void SetVolume(float volume) OVERRIDE {
    103     DCHECK(thread_checker_.CalledOnValidThread());
    104     DCHECK(volume >= 0.0f && volume <= 1.0f);
    105     playing_state_.set_volume(volume);
    106     on_play_state_changed_.Run(media_stream_, &playing_state_);
    107   }
    108 
    109   virtual base::TimeDelta GetCurrentRenderTime() const OVERRIDE {
    110     DCHECK(thread_checker_.CalledOnValidThread());
    111     return delegate_->GetCurrentRenderTime();
    112   }
    113 
    114   virtual bool IsLocalRenderer() const OVERRIDE {
    115     DCHECK(thread_checker_.CalledOnValidThread());
    116     return delegate_->IsLocalRenderer();
    117   }
    118 
    119  private:
    120   base::ThreadChecker thread_checker_;
    121   const scoped_refptr<MediaStreamAudioRenderer> delegate_;
    122   const scoped_refptr<webrtc::MediaStreamInterface> media_stream_;
    123   bool started_;
    124   WebRtcAudioRenderer::PlayingState playing_state_;
    125   OnPlayStateChanged on_play_state_changed_;
    126 };
    127 
    128 // Returns either AudioParameters::NO_EFFECTS or AudioParameters::DUCKING
    129 // depending on whether or not an input element is currently open with
    130 // ducking enabled.
    131 int GetCurrentDuckingFlag(int render_frame_id) {
    132   RenderFrameImpl* const frame =
    133       RenderFrameImpl::FromRoutingID(render_frame_id);
    134   MediaStreamDispatcher* const dispatcher = frame ?
    135       frame->GetMediaStreamDispatcher() : NULL;
    136   if (dispatcher && dispatcher->IsAudioDuckingActive()) {
    137     return media::AudioParameters::DUCKING;
    138   }
    139 
    140   return media::AudioParameters::NO_EFFECTS;
    141 }
    142 
    143 // Helper method to get platform specific optimal buffer size.
    144 int GetOptimalBufferSize(int sample_rate, int hardware_buffer_size) {
    145   // Use native hardware buffer size as default. On Windows, we strive to open
    146   // up using this native hardware buffer size to achieve best
    147   // possible performance and to ensure that no FIFO is needed on the browser
    148   // side to match the client request. That is why there is no #if case for
    149   // Windows below.
    150   int frames_per_buffer = hardware_buffer_size;
    151 
    152 #if defined(OS_LINUX) || defined(OS_MACOSX)
    153   // On Linux and MacOS, the low level IO implementations on the browser side
    154   // supports all buffer size the clients want. We use the native peer
    155   // connection buffer size (10ms) to achieve best possible performance.
    156   frames_per_buffer = sample_rate / 100;
    157 #elif defined(OS_ANDROID)
    158   // TODO(henrika): Keep tuning this scheme and espcicially for low-latency
    159   // cases. Might not be possible to come up with the perfect solution using
    160   // the render side only.
    161   int frames_per_10ms = sample_rate / 100;
    162   if (frames_per_buffer < 2 * frames_per_10ms) {
    163     // Examples of low-latency frame sizes and the resulting |buffer_size|:
    164     //  Nexus 7     : 240 audio frames => 2*480 = 960
    165     //  Nexus 10    : 256              => 2*441 = 882
    166     //  Galaxy Nexus: 144              => 2*441 = 882
    167     frames_per_buffer = 2 * frames_per_10ms;
    168     DVLOG(1) << "Low-latency output detected on Android";
    169   }
    170 #endif
    171 
    172   DVLOG(1) << "Using sink output buffer size: " << frames_per_buffer;
    173   return frames_per_buffer;
    174 }
    175 
    176 }  // namespace
    177 
    178 WebRtcAudioRenderer::WebRtcAudioRenderer(
    179     const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
    180     int source_render_view_id,
    181     int source_render_frame_id,
    182     int session_id,
    183     int sample_rate,
    184     int frames_per_buffer)
    185     : state_(UNINITIALIZED),
    186       source_render_view_id_(source_render_view_id),
    187       source_render_frame_id_(source_render_frame_id),
    188       session_id_(session_id),
    189       media_stream_(media_stream),
    190       source_(NULL),
    191       play_ref_count_(0),
    192       start_ref_count_(0),
    193       audio_delay_milliseconds_(0),
    194       fifo_delay_milliseconds_(0),
    195       sink_params_(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
    196                    media::CHANNEL_LAYOUT_STEREO, sample_rate, 16,
    197                    frames_per_buffer,
    198                    GetCurrentDuckingFlag(source_render_frame_id)),
    199       render_callback_count_(0) {
    200   WebRtcLogMessage(base::StringPrintf(
    201       "WAR::WAR. source_render_view_id=%d"
    202       ", session_id=%d, sample_rate=%d, frames_per_buffer=%d, effects=%i",
    203       source_render_view_id,
    204       session_id,
    205       sample_rate,
    206       frames_per_buffer,
    207       sink_params_.effects()));
    208 }
    209 
    210 WebRtcAudioRenderer::~WebRtcAudioRenderer() {
    211   DCHECK(thread_checker_.CalledOnValidThread());
    212   DCHECK_EQ(state_, UNINITIALIZED);
    213 }
    214 
    215 bool WebRtcAudioRenderer::Initialize(WebRtcAudioRendererSource* source) {
    216   DVLOG(1) << "WebRtcAudioRenderer::Initialize()";
    217   DCHECK(thread_checker_.CalledOnValidThread());
    218   base::AutoLock auto_lock(lock_);
    219   DCHECK_EQ(state_, UNINITIALIZED);
    220   DCHECK(source);
    221   DCHECK(!sink_.get());
    222   DCHECK(!source_);
    223 
    224   // WebRTC does not yet support higher rates than 96000 on the client side
    225   // and 48000 is the preferred sample rate. Therefore, if 192000 is detected,
    226   // we change the rate to 48000 instead. The consequence is that the native
    227   // layer will be opened up at 192kHz but WebRTC will provide data at 48kHz
    228   // which will then be resampled by the audio converted on the browser side
    229   // to match the native audio layer.
    230   int sample_rate = sink_params_.sample_rate();
    231   DVLOG(1) << "Audio output hardware sample rate: " << sample_rate;
    232   if (sample_rate == 192000) {
    233     DVLOG(1) << "Resampling from 48000 to 192000 is required";
    234     sample_rate = 48000;
    235   }
    236   media::AudioSampleRate asr;
    237   if (media::ToAudioSampleRate(sample_rate, &asr)) {
    238     UMA_HISTOGRAM_ENUMERATION(
    239         "WebRTC.AudioOutputSampleRate", asr, media::kAudioSampleRateMax + 1);
    240   } else {
    241     UMA_HISTOGRAM_COUNTS("WebRTC.AudioOutputSampleRateUnexpected",
    242                          sample_rate);
    243   }
    244 
    245   // Set up audio parameters for the source, i.e., the WebRTC client.
    246 
    247   // The WebRTC client only supports multiples of 10ms as buffer size where
    248   // 10ms is preferred for lowest possible delay.
    249   media::AudioParameters source_params;
    250   const int frames_per_10ms = (sample_rate / 100);
    251   DVLOG(1) << "Using WebRTC output buffer size: " << frames_per_10ms;
    252 
    253   source_params.Reset(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
    254                       sink_params_.channel_layout(), sink_params_.channels(),
    255                       sample_rate, 16, frames_per_10ms);
    256 
    257   const int frames_per_buffer =
    258       GetOptimalBufferSize(sample_rate, sink_params_.frames_per_buffer());
    259 
    260   sink_params_.Reset(sink_params_.format(), sink_params_.channel_layout(),
    261                      sink_params_.channels(), sample_rate, 16,
    262                      frames_per_buffer);
    263 
    264   // Create a FIFO if re-buffering is required to match the source input with
    265   // the sink request. The source acts as provider here and the sink as
    266   // consumer.
    267   fifo_delay_milliseconds_ = 0;
    268   if (source_params.frames_per_buffer() != sink_params_.frames_per_buffer()) {
    269     DVLOG(1) << "Rebuffering from " << source_params.frames_per_buffer()
    270              << " to " << sink_params_.frames_per_buffer();
    271     audio_fifo_.reset(new media::AudioPullFifo(
    272         source_params.channels(),
    273         source_params.frames_per_buffer(),
    274         base::Bind(
    275             &WebRtcAudioRenderer::SourceCallback,
    276             base::Unretained(this))));
    277 
    278     if (sink_params_.frames_per_buffer() > source_params.frames_per_buffer()) {
    279       int frame_duration_milliseconds = base::Time::kMillisecondsPerSecond /
    280           static_cast<double>(source_params.sample_rate());
    281       fifo_delay_milliseconds_ = (sink_params_.frames_per_buffer() -
    282         source_params.frames_per_buffer()) * frame_duration_milliseconds;
    283     }
    284   }
    285 
    286   source_ = source;
    287 
    288   // Configure the audio rendering client and start rendering.
    289   sink_ = AudioDeviceFactory::NewOutputDevice(
    290       source_render_view_id_, source_render_frame_id_);
    291 
    292   DCHECK_GE(session_id_, 0);
    293   sink_->InitializeWithSessionId(sink_params_, this, session_id_);
    294 
    295   sink_->Start();
    296 
    297   // User must call Play() before any audio can be heard.
    298   state_ = PAUSED;
    299 
    300   return true;
    301 }
    302 
    303 scoped_refptr<MediaStreamAudioRenderer>
    304 WebRtcAudioRenderer::CreateSharedAudioRendererProxy(
    305     const scoped_refptr<webrtc::MediaStreamInterface>& media_stream) {
    306   content::SharedAudioRenderer::OnPlayStateChanged on_play_state_changed =
    307       base::Bind(&WebRtcAudioRenderer::OnPlayStateChanged, this);
    308   return new SharedAudioRenderer(this, media_stream, on_play_state_changed);
    309 }
    310 
    311 bool WebRtcAudioRenderer::IsStarted() const {
    312   DCHECK(thread_checker_.CalledOnValidThread());
    313   return start_ref_count_ != 0;
    314 }
    315 
    316 void WebRtcAudioRenderer::Start() {
    317   DVLOG(1) << "WebRtcAudioRenderer::Start()";
    318   DCHECK(thread_checker_.CalledOnValidThread());
    319   ++start_ref_count_;
    320 }
    321 
    322 void WebRtcAudioRenderer::Play() {
    323   DVLOG(1) << "WebRtcAudioRenderer::Play()";
    324   DCHECK(thread_checker_.CalledOnValidThread());
    325 
    326   if (playing_state_.playing())
    327     return;
    328 
    329   playing_state_.set_playing(true);
    330   render_callback_count_ = 0;
    331 
    332   OnPlayStateChanged(media_stream_, &playing_state_);
    333 }
    334 
    335 void WebRtcAudioRenderer::EnterPlayState() {
    336   DVLOG(1) << "WebRtcAudioRenderer::EnterPlayState()";
    337   DCHECK(thread_checker_.CalledOnValidThread());
    338   DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?";
    339   base::AutoLock auto_lock(lock_);
    340   if (state_ == UNINITIALIZED)
    341     return;
    342 
    343   DCHECK(play_ref_count_ == 0 || state_ == PLAYING);
    344   ++play_ref_count_;
    345 
    346   if (state_ != PLAYING) {
    347     state_ = PLAYING;
    348 
    349     if (audio_fifo_) {
    350       audio_delay_milliseconds_ = 0;
    351       audio_fifo_->Clear();
    352     }
    353   }
    354 }
    355 
    356 void WebRtcAudioRenderer::Pause() {
    357   DVLOG(1) << "WebRtcAudioRenderer::Pause()";
    358   DCHECK(thread_checker_.CalledOnValidThread());
    359   if (!playing_state_.playing())
    360     return;
    361 
    362   playing_state_.set_playing(false);
    363 
    364   OnPlayStateChanged(media_stream_, &playing_state_);
    365 }
    366 
    367 void WebRtcAudioRenderer::EnterPauseState() {
    368   DVLOG(1) << "WebRtcAudioRenderer::EnterPauseState()";
    369   DCHECK(thread_checker_.CalledOnValidThread());
    370   DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?";
    371   base::AutoLock auto_lock(lock_);
    372   if (state_ == UNINITIALIZED)
    373     return;
    374 
    375   DCHECK_EQ(state_, PLAYING);
    376   DCHECK_GT(play_ref_count_, 0);
    377   if (!--play_ref_count_)
    378     state_ = PAUSED;
    379 }
    380 
    381 void WebRtcAudioRenderer::Stop() {
    382   DVLOG(1) << "WebRtcAudioRenderer::Stop()";
    383   DCHECK(thread_checker_.CalledOnValidThread());
    384   {
    385     base::AutoLock auto_lock(lock_);
    386     if (state_ == UNINITIALIZED)
    387       return;
    388 
    389     if (--start_ref_count_)
    390       return;
    391 
    392     DVLOG(1) << "Calling RemoveAudioRenderer and Stop().";
    393 
    394     source_->RemoveAudioRenderer(this);
    395     source_ = NULL;
    396     state_ = UNINITIALIZED;
    397   }
    398 
    399   // Make sure to stop the sink while _not_ holding the lock since the Render()
    400   // callback may currently be executing and try to grab the lock while we're
    401   // stopping the thread on which it runs.
    402   sink_->Stop();
    403 }
    404 
    405 void WebRtcAudioRenderer::SetVolume(float volume) {
    406   DCHECK(thread_checker_.CalledOnValidThread());
    407   DCHECK(volume >= 0.0f && volume <= 1.0f);
    408 
    409   playing_state_.set_volume(volume);
    410   OnPlayStateChanged(media_stream_, &playing_state_);
    411 }
    412 
    413 base::TimeDelta WebRtcAudioRenderer::GetCurrentRenderTime() const {
    414   DCHECK(thread_checker_.CalledOnValidThread());
    415   base::AutoLock auto_lock(lock_);
    416   return current_time_;
    417 }
    418 
    419 bool WebRtcAudioRenderer::IsLocalRenderer() const {
    420   return false;
    421 }
    422 
    423 int WebRtcAudioRenderer::Render(media::AudioBus* audio_bus,
    424                                 int audio_delay_milliseconds) {
    425   base::AutoLock auto_lock(lock_);
    426   if (!source_)
    427     return 0;
    428 
    429   DVLOG(2) << "WebRtcAudioRenderer::Render()";
    430   DVLOG(2) << "audio_delay_milliseconds: " << audio_delay_milliseconds;
    431 
    432   audio_delay_milliseconds_ = audio_delay_milliseconds;
    433 
    434   if (audio_fifo_)
    435     audio_fifo_->Consume(audio_bus, audio_bus->frames());
    436   else
    437     SourceCallback(0, audio_bus);
    438 
    439   return (state_ == PLAYING) ? audio_bus->frames() : 0;
    440 }
    441 
    442 void WebRtcAudioRenderer::OnRenderError() {
    443   NOTIMPLEMENTED();
    444   LOG(ERROR) << "OnRenderError()";
    445 }
    446 
    447 // Called by AudioPullFifo when more data is necessary.
    448 void WebRtcAudioRenderer::SourceCallback(
    449     int fifo_frame_delay, media::AudioBus* audio_bus) {
    450   base::TimeTicks start_time = base::TimeTicks::Now() ;
    451   DVLOG(2) << "WebRtcAudioRenderer::SourceCallback("
    452            << fifo_frame_delay << ", "
    453            << audio_bus->frames() << ")";
    454 
    455   int output_delay_milliseconds = audio_delay_milliseconds_;
    456   output_delay_milliseconds += fifo_delay_milliseconds_;
    457   DVLOG(2) << "output_delay_milliseconds: " << output_delay_milliseconds;
    458 
    459   // We need to keep render data for the |source_| regardless of |state_|,
    460   // otherwise the data will be buffered up inside |source_|.
    461   source_->RenderData(audio_bus, sink_params_.sample_rate(),
    462                       output_delay_milliseconds,
    463                       &current_time_);
    464 
    465   // Avoid filling up the audio bus if we are not playing; instead
    466   // return here and ensure that the returned value in Render() is 0.
    467   if (state_ != PLAYING)
    468     audio_bus->Zero();
    469 
    470   if (++render_callback_count_ == kNumCallbacksBetweenRenderTimeHistograms) {
    471     base::TimeDelta elapsed = base::TimeTicks::Now() - start_time;
    472     render_callback_count_ = 0;
    473     UMA_HISTOGRAM_TIMES("WebRTC.AudioRenderTimes", elapsed);
    474   }
    475 }
    476 
    477 void WebRtcAudioRenderer::UpdateSourceVolume(
    478     webrtc::AudioSourceInterface* source) {
    479   DCHECK(thread_checker_.CalledOnValidThread());
    480 
    481   // Note: If there are no playing audio renderers, then the volume will be
    482   // set to 0.0.
    483   float volume = 0.0f;
    484 
    485   SourcePlayingStates::iterator entry = source_playing_states_.find(source);
    486   if (entry != source_playing_states_.end()) {
    487     PlayingStates& states = entry->second;
    488     for (PlayingStates::const_iterator it = states.begin();
    489          it != states.end(); ++it) {
    490       if ((*it)->playing())
    491         volume += (*it)->volume();
    492     }
    493   }
    494 
    495   // The valid range for volume scaling of a remote webrtc source is
    496   // 0.0-10.0 where 1.0 is no attenuation/boost.
    497   DCHECK(volume >= 0.0f);
    498   if (volume > 10.0f)
    499     volume = 10.0f;
    500 
    501   DVLOG(1) << "Setting remote source volume: " << volume;
    502   source->SetVolume(volume);
    503 }
    504 
    505 bool WebRtcAudioRenderer::AddPlayingState(
    506     webrtc::AudioSourceInterface* source,
    507     PlayingState* state) {
    508   DCHECK(thread_checker_.CalledOnValidThread());
    509   DCHECK(state->playing());
    510   // Look up or add the |source| to the map.
    511   PlayingStates& array = source_playing_states_[source];
    512   if (std::find(array.begin(), array.end(), state) != array.end())
    513     return false;
    514 
    515   array.push_back(state);
    516 
    517   return true;
    518 }
    519 
    520 bool WebRtcAudioRenderer::RemovePlayingState(
    521     webrtc::AudioSourceInterface* source,
    522     PlayingState* state) {
    523   DCHECK(thread_checker_.CalledOnValidThread());
    524   DCHECK(!state->playing());
    525   SourcePlayingStates::iterator found = source_playing_states_.find(source);
    526   if (found == source_playing_states_.end())
    527     return false;
    528 
    529   PlayingStates& array = found->second;
    530   PlayingStates::iterator state_it =
    531       std::find(array.begin(), array.end(), state);
    532   if (state_it == array.end())
    533     return false;
    534 
    535   array.erase(state_it);
    536 
    537   if (array.empty())
    538     source_playing_states_.erase(found);
    539 
    540   return true;
    541 }
    542 
    543 void WebRtcAudioRenderer::OnPlayStateChanged(
    544     const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
    545     PlayingState* state) {
    546   webrtc::AudioTrackVector tracks(media_stream->GetAudioTracks());
    547   for (webrtc::AudioTrackVector::iterator it = tracks.begin();
    548        it != tracks.end(); ++it) {
    549     webrtc::AudioSourceInterface* source = (*it)->GetSource();
    550     DCHECK(source);
    551     if (!state->playing()) {
    552       if (RemovePlayingState(source, state))
    553         EnterPauseState();
    554     } else if (AddPlayingState(source, state)) {
    555       EnterPlayState();
    556     }
    557     UpdateSourceVolume(source);
    558   }
    559 }
    560 
    561 }  // namespace content
    562