Home | History | Annotate | Download | only in media 
      1 // Copyright 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "content/renderer/media/webrtc_audio_device_impl.h"
      6 
      7 #include "base/bind.h"
      8 #include "base/metrics/histogram.h"
      9 #include "base/strings/string_util.h"
     10 #include "base/win/windows_version.h"
     11 #include "content/renderer/media/media_stream_audio_processor.h"
     12 #include "content/renderer/media/webrtc_audio_capturer.h"
     13 #include "content/renderer/media/webrtc_audio_renderer.h"
     14 #include "content/renderer/render_thread_impl.h"
     15 #include "media/audio/audio_parameters.h"
     16 #include "media/audio/sample_rates.h"
     17 
     18 using media::AudioParameters;
     19 using media::ChannelLayout;
     20 
     21 namespace content {
     22 
     23 WebRtcAudioDeviceImpl::WebRtcAudioDeviceImpl()
     24     : ref_count_(0),
     25       audio_transport_callback_(NULL),
     26       input_delay_ms_(0),
     27       output_delay_ms_(0),
     28       initialized_(false),
     29       playing_(false),
     30       recording_(false),
     31       microphone_volume_(0),
     32       is_audio_track_processing_enabled_(
     33           MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled()) {
     34   DVLOG(1) << "WebRtcAudioDeviceImpl::WebRtcAudioDeviceImpl()";
     35 }
     36 
     37 WebRtcAudioDeviceImpl::~WebRtcAudioDeviceImpl() {
     38   DVLOG(1) << "WebRtcAudioDeviceImpl::~WebRtcAudioDeviceImpl()";
     39   DCHECK(thread_checker_.CalledOnValidThread());
     40   Terminate();
     41 }
     42 
     43 int32_t WebRtcAudioDeviceImpl::AddRef() {
     44   DCHECK(thread_checker_.CalledOnValidThread());
     45   return base::subtle::Barrier_AtomicIncrement(&ref_count_, 1);
     46 }
     47 
     48 int32_t WebRtcAudioDeviceImpl::Release() {
     49   DCHECK(thread_checker_.CalledOnValidThread());
     50   int ret = base::subtle::Barrier_AtomicIncrement(&ref_count_, -1);
     51   if (ret == 0) {
     52     delete this;
     53   }
     54   return ret;
     55 }
     56 int WebRtcAudioDeviceImpl::OnData(const int16* audio_data,
     57                                   int sample_rate,
     58                                   int number_of_channels,
     59                                   int number_of_frames,
     60                                   const std::vector<int>& channels,
     61                                   int audio_delay_milliseconds,
     62                                   int current_volume,
     63                                   bool need_audio_processing,
     64                                   bool key_pressed) {
     65   int total_delay_ms = 0;
     66   {
     67     base::AutoLock auto_lock(lock_);
     68     // Return immediately when not recording or |channels| is empty.
     69     // See crbug.com/274017: renderer crash dereferencing invalid channels[0].
     70     if (!recording_ || channels.empty())
     71       return 0;
     72 
     73     // Store the reported audio delay locally.
     74     input_delay_ms_ = audio_delay_milliseconds;
     75     total_delay_ms = input_delay_ms_ + output_delay_ms_;
     76     DVLOG(2) << "total delay: " << input_delay_ms_ + output_delay_ms_;
     77   }
     78 
     79   // Write audio frames in blocks of 10 milliseconds to the registered
     80   // webrtc::AudioTransport sink. Keep writing until our internal byte
     81   // buffer is empty.
     82   const int16* audio_buffer = audio_data;
     83   const int frames_per_10_ms = (sample_rate / 100);
     84   CHECK_EQ(number_of_frames % frames_per_10_ms, 0);
     85   int accumulated_audio_frames = 0;
     86   uint32_t new_volume = 0;
     87 
     88   // The lock here is to protect a race in the resampler inside webrtc when
     89   // there are more than one input stream calling OnData(), which can happen
     90   // when the users setup two getUserMedia, one for the microphone, another
     91   // for WebAudio. Currently we don't have a better way to fix it except for
     92   // adding a lock here to sequence the call.
     93   // TODO(xians): Remove this workaround after we move the
     94   // webrtc::AudioProcessing module to Chrome. See http://crbug/264611 for
     95   // details.
     96   base::AutoLock auto_lock(capture_callback_lock_);
     97   while (accumulated_audio_frames < number_of_frames) {
     98     // Deliver 10ms of recorded 16-bit linear PCM audio.
     99     int new_mic_level = audio_transport_callback_->OnDataAvailable(
    100         &channels[0],
    101         channels.size(),
    102         audio_buffer,
    103         sample_rate,
    104         number_of_channels,
    105         frames_per_10_ms,
    106         total_delay_ms,
    107         current_volume,
    108         key_pressed,
    109         need_audio_processing);
    110 
    111     accumulated_audio_frames += frames_per_10_ms;
    112     audio_buffer += frames_per_10_ms * number_of_channels;
    113 
    114     // The latest non-zero new microphone level will be returned.
    115     if (new_mic_level)
    116       new_volume = new_mic_level;
    117   }
    118 
    119   return new_volume;
    120 }
    121 
    122 void WebRtcAudioDeviceImpl::OnSetFormat(
    123     const media::AudioParameters& params) {
    124   DVLOG(1) << "WebRtcAudioDeviceImpl::OnSetFormat()";
    125 }
    126 
    127 void WebRtcAudioDeviceImpl::RenderData(media::AudioBus* audio_bus,
    128                                        int sample_rate,
    129                                        int audio_delay_milliseconds,
    130                                        base::TimeDelta* current_time) {
    131   render_buffer_.resize(audio_bus->frames() * audio_bus->channels());
    132 
    133   {
    134     base::AutoLock auto_lock(lock_);
    135     DCHECK(audio_transport_callback_);
    136     // Store the reported audio delay locally.
    137     output_delay_ms_ = audio_delay_milliseconds;
    138   }
    139 
    140   int frames_per_10_ms = (sample_rate / 100);
    141   int bytes_per_sample = sizeof(render_buffer_[0]);
    142   const int bytes_per_10_ms =
    143       audio_bus->channels() * frames_per_10_ms * bytes_per_sample;
    144   DCHECK_EQ(audio_bus->frames() % frames_per_10_ms, 0);
    145 
    146   // Get audio frames in blocks of 10 milliseconds from the registered
    147   // webrtc::AudioTransport source. Keep reading until our internal buffer
    148   // is full.
    149   uint32_t num_audio_frames = 0;
    150   int accumulated_audio_frames = 0;
    151   int16* audio_data = &render_buffer_[0];
    152   while (accumulated_audio_frames < audio_bus->frames()) {
    153     // Get 10ms and append output to temporary byte buffer.
    154     int64_t elapsed_time_ms = -1;
    155     int64_t ntp_time_ms = -1;
    156     if (is_audio_track_processing_enabled_) {
    157       // When audio processing is enabled in the audio track, we use
    158       // PullRenderData() instead of NeedMorePlayData() to avoid passing the
    159       // render data to the APM in WebRTC as reference signal for echo
    160       // cancellation.
    161       static const int kBitsPerByte = 8;
    162       audio_transport_callback_->PullRenderData(bytes_per_sample * kBitsPerByte,
    163                                                 sample_rate,
    164                                                 audio_bus->channels(),
    165                                                 frames_per_10_ms,
    166                                                 audio_data,
    167                                                 &elapsed_time_ms,
    168                                                 &ntp_time_ms);
    169       accumulated_audio_frames += frames_per_10_ms;
    170     } else {
    171       // TODO(xians): Remove the following code after the APM in WebRTC is
    172       // deprecated.
    173       audio_transport_callback_->NeedMorePlayData(frames_per_10_ms,
    174                                                   bytes_per_sample,
    175                                                   audio_bus->channels(),
    176                                                   sample_rate,
    177                                                   audio_data,
    178                                                   num_audio_frames,
    179                                                   &elapsed_time_ms,
    180                                                   &ntp_time_ms);
    181       accumulated_audio_frames += num_audio_frames;
    182     }
    183     if (elapsed_time_ms >= 0) {
    184       *current_time = base::TimeDelta::FromMilliseconds(elapsed_time_ms);
    185     }
    186     audio_data += bytes_per_10_ms;
    187   }
    188 
    189   // De-interleave each channel and convert to 32-bit floating-point
    190   // with nominal range -1.0 -> +1.0 to match the callback format.
    191   audio_bus->FromInterleaved(&render_buffer_[0],
    192                              audio_bus->frames(),
    193                              bytes_per_sample);
    194 
    195   // Pass the render data to the playout sinks.
    196   base::AutoLock auto_lock(lock_);
    197   for (PlayoutDataSinkList::const_iterator it = playout_sinks_.begin();
    198        it != playout_sinks_.end(); ++it) {
    199     (*it)->OnPlayoutData(audio_bus, sample_rate, audio_delay_milliseconds);
    200   }
    201 }
    202 
    203 void WebRtcAudioDeviceImpl::RemoveAudioRenderer(WebRtcAudioRenderer* renderer) {
    204   DCHECK(thread_checker_.CalledOnValidThread());
    205   DCHECK_EQ(renderer, renderer_);
    206   base::AutoLock auto_lock(lock_);
    207   // Notify the playout sink of the change.
    208   for (PlayoutDataSinkList::const_iterator it = playout_sinks_.begin();
    209        it != playout_sinks_.end(); ++it) {
    210     (*it)->OnPlayoutDataSourceChanged();
    211   }
    212 
    213   renderer_ = NULL;
    214   playing_ = false;
    215 }
    216 
    217 int32_t WebRtcAudioDeviceImpl::RegisterAudioCallback(
    218     webrtc::AudioTransport* audio_callback) {
    219   DVLOG(1) << "WebRtcAudioDeviceImpl::RegisterAudioCallback()";
    220   DCHECK(thread_checker_.CalledOnValidThread());
    221   DCHECK_EQ(audio_transport_callback_ == NULL, audio_callback != NULL);
    222   audio_transport_callback_ = audio_callback;
    223   return 0;
    224 }
    225 
    226 int32_t WebRtcAudioDeviceImpl::Init() {
    227   DVLOG(1) << "WebRtcAudioDeviceImpl::Init()";
    228   DCHECK(thread_checker_.CalledOnValidThread());
    229 
    230   // We need to return a success to continue the initialization of WebRtc VoE
    231   // because failure on the capturer_ initialization should not prevent WebRTC
    232   // from working. See issue http://crbug.com/144421 for details.
    233   initialized_ = true;
    234 
    235   return 0;
    236 }
    237 
    238 int32_t WebRtcAudioDeviceImpl::Terminate() {
    239   DVLOG(1) << "WebRtcAudioDeviceImpl::Terminate()";
    240   DCHECK(thread_checker_.CalledOnValidThread());
    241 
    242   // Calling Terminate() multiple times in a row is OK.
    243   if (!initialized_)
    244     return 0;
    245 
    246   StopRecording();
    247   StopPlayout();
    248 
    249   DCHECK(!renderer_.get() || !renderer_->IsStarted())
    250       << "The shared audio renderer shouldn't be running";
    251 
    252   // Stop all the capturers to ensure no further OnData() and
    253   // RemoveAudioCapturer() callback.
    254   // Cache the capturers in a local list since WebRtcAudioCapturer::Stop()
    255   // will trigger RemoveAudioCapturer() callback.
    256   CapturerList capturers;
    257   capturers.swap(capturers_);
    258   for (CapturerList::const_iterator iter = capturers.begin();
    259        iter != capturers.end(); ++iter) {
    260     (*iter)->Stop();
    261   }
    262 
    263   initialized_ = false;
    264   return 0;
    265 }
    266 
    267 bool WebRtcAudioDeviceImpl::Initialized() const {
    268   return initialized_;
    269 }
    270 
    271 int32_t WebRtcAudioDeviceImpl::PlayoutIsAvailable(bool* available) {
    272   *available = initialized_;
    273   return 0;
    274 }
    275 
    276 bool WebRtcAudioDeviceImpl::PlayoutIsInitialized() const {
    277   return initialized_;
    278 }
    279 
    280 int32_t WebRtcAudioDeviceImpl::RecordingIsAvailable(bool* available) {
    281   *available = (!capturers_.empty());
    282   return 0;
    283 }
    284 
    285 bool WebRtcAudioDeviceImpl::RecordingIsInitialized() const {
    286   DVLOG(1) << "WebRtcAudioDeviceImpl::RecordingIsInitialized()";
    287   DCHECK(thread_checker_.CalledOnValidThread());
    288   return (!capturers_.empty());
    289 }
    290 
    291 int32_t WebRtcAudioDeviceImpl::StartPlayout() {
    292   DVLOG(1) << "WebRtcAudioDeviceImpl::StartPlayout()";
    293   LOG_IF(ERROR, !audio_transport_callback_) << "Audio transport is missing";
    294   {
    295     base::AutoLock auto_lock(lock_);
    296     if (!audio_transport_callback_)
    297       return 0;
    298   }
    299 
    300   if (playing_) {
    301     // webrtc::VoiceEngine assumes that it is OK to call Start() twice and
    302     // that the call is ignored the second time.
    303     return 0;
    304   }
    305 
    306   playing_ = true;
    307   return 0;
    308 }
    309 
    310 int32_t WebRtcAudioDeviceImpl::StopPlayout() {
    311   DVLOG(1) << "WebRtcAudioDeviceImpl::StopPlayout()";
    312   if (!playing_) {
    313     // webrtc::VoiceEngine assumes that it is OK to call Stop() just in case.
    314     return 0;
    315   }
    316 
    317   playing_ = false;
    318   return 0;
    319 }
    320 
    321 bool WebRtcAudioDeviceImpl::Playing() const {
    322   return playing_;
    323 }
    324 
    325 int32_t WebRtcAudioDeviceImpl::StartRecording() {
    326   DVLOG(1) << "WebRtcAudioDeviceImpl::StartRecording()";
    327   DCHECK(initialized_);
    328   LOG_IF(ERROR, !audio_transport_callback_) << "Audio transport is missing";
    329   if (!audio_transport_callback_) {
    330     return -1;
    331   }
    332 
    333   {
    334     base::AutoLock auto_lock(lock_);
    335     if (recording_)
    336       return 0;
    337 
    338     recording_ = true;
    339   }
    340 
    341   return 0;
    342 }
    343 
    344 int32_t WebRtcAudioDeviceImpl::StopRecording() {
    345   DVLOG(1) << "WebRtcAudioDeviceImpl::StopRecording()";
    346   {
    347     base::AutoLock auto_lock(lock_);
    348     if (!recording_)
    349       return 0;
    350 
    351     recording_ = false;
    352   }
    353 
    354   return 0;
    355 }
    356 
    357 bool WebRtcAudioDeviceImpl::Recording() const {
    358   base::AutoLock auto_lock(lock_);
    359   return recording_;
    360 }
    361 
    362 int32_t WebRtcAudioDeviceImpl::SetMicrophoneVolume(uint32_t volume) {
    363   DVLOG(1) << "WebRtcAudioDeviceImpl::SetMicrophoneVolume(" << volume << ")";
    364   DCHECK(initialized_);
    365 
    366   // Only one microphone is supported at the moment, which is represented by
    367   // the default capturer.
    368   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
    369   if (!capturer.get())
    370     return -1;
    371 
    372   capturer->SetVolume(volume);
    373   return 0;
    374 }
    375 
    376 // TODO(henrika): sort out calling thread once we start using this API.
    377 int32_t WebRtcAudioDeviceImpl::MicrophoneVolume(uint32_t* volume) const {
    378   DVLOG(1) << "WebRtcAudioDeviceImpl::MicrophoneVolume()";
    379   // We only support one microphone now, which is accessed via the default
    380   // capturer.
    381   DCHECK(initialized_);
    382   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
    383   if (!capturer.get())
    384     return -1;
    385 
    386   *volume = static_cast<uint32_t>(capturer->Volume());
    387 
    388   return 0;
    389 }
    390 
    391 int32_t WebRtcAudioDeviceImpl::MaxMicrophoneVolume(uint32_t* max_volume) const {
    392   DCHECK(initialized_);
    393   *max_volume = kMaxVolumeLevel;
    394   return 0;
    395 }
    396 
    397 int32_t WebRtcAudioDeviceImpl::MinMicrophoneVolume(uint32_t* min_volume) const {
    398   *min_volume = 0;
    399   return 0;
    400 }
    401 
    402 int32_t WebRtcAudioDeviceImpl::StereoPlayoutIsAvailable(bool* available) const {
    403   DCHECK(initialized_);
    404   *available = renderer_ && renderer_->channels() == 2;
    405   return 0;
    406 }
    407 
    408 int32_t WebRtcAudioDeviceImpl::StereoRecordingIsAvailable(
    409     bool* available) const {
    410   DCHECK(initialized_);
    411   // TODO(xians): These kind of hardware methods do not make much sense since we
    412   // support multiple sources. Remove or figure out new APIs for such methods.
    413   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
    414   if (!capturer.get())
    415     return -1;
    416 
    417   *available = (capturer->source_audio_parameters().channels() == 2);
    418   return 0;
    419 }
    420 
    421 int32_t WebRtcAudioDeviceImpl::PlayoutDelay(uint16_t* delay_ms) const {
    422   base::AutoLock auto_lock(lock_);
    423   *delay_ms = static_cast<uint16_t>(output_delay_ms_);
    424   return 0;
    425 }
    426 
    427 int32_t WebRtcAudioDeviceImpl::RecordingDelay(uint16_t* delay_ms) const {
    428   base::AutoLock auto_lock(lock_);
    429   *delay_ms = static_cast<uint16_t>(input_delay_ms_);
    430   return 0;
    431 }
    432 
    433 int32_t WebRtcAudioDeviceImpl::RecordingSampleRate(
    434     uint32_t* sample_rate) const {
    435   // We use the default capturer as the recording sample rate.
    436   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
    437   if (!capturer.get())
    438     return -1;
    439 
    440   *sample_rate = static_cast<uint32_t>(
    441       capturer->source_audio_parameters().sample_rate());
    442   return 0;
    443 }
    444 
    445 int32_t WebRtcAudioDeviceImpl::PlayoutSampleRate(
    446     uint32_t* sample_rate) const {
    447   *sample_rate = renderer_ ? renderer_->sample_rate() : 0;
    448   return 0;
    449 }
    450 
    451 bool WebRtcAudioDeviceImpl::SetAudioRenderer(WebRtcAudioRenderer* renderer) {
    452   DCHECK(thread_checker_.CalledOnValidThread());
    453   DCHECK(renderer);
    454 
    455   base::AutoLock auto_lock(lock_);
    456   if (renderer_.get())
    457     return false;
    458 
    459   if (!renderer->Initialize(this))
    460     return false;
    461 
    462   renderer_ = renderer;
    463   return true;
    464 }
    465 
    466 void WebRtcAudioDeviceImpl::AddAudioCapturer(
    467     const scoped_refptr<WebRtcAudioCapturer>& capturer) {
    468   DVLOG(1) << "WebRtcAudioDeviceImpl::AddAudioCapturer()";
    469   DCHECK(thread_checker_.CalledOnValidThread());
    470   DCHECK(capturer.get());
    471   DCHECK(!capturer->device_id().empty());
    472   {
    473     base::AutoLock auto_lock(lock_);
    474     DCHECK(std::find(capturers_.begin(), capturers_.end(), capturer) ==
    475         capturers_.end());
    476     capturers_.push_back(capturer);
    477   }
    478 }
    479 
    480 void WebRtcAudioDeviceImpl::RemoveAudioCapturer(
    481     const scoped_refptr<WebRtcAudioCapturer>& capturer) {
    482   DVLOG(1) << "WebRtcAudioDeviceImpl::AddAudioCapturer()";
    483   DCHECK(thread_checker_.CalledOnValidThread());
    484   DCHECK(capturer.get());
    485   base::AutoLock auto_lock(lock_);
    486   capturers_.remove(capturer);
    487 }
    488 
    489 scoped_refptr<WebRtcAudioCapturer>
    490 WebRtcAudioDeviceImpl::GetDefaultCapturer() const {
    491   base::AutoLock auto_lock(lock_);
    492   // Use the last |capturer| which is from the latest getUserMedia call as
    493   // the default capture device.
    494   return capturers_.empty() ? NULL : capturers_.back();
    495 }
    496 
    497 void WebRtcAudioDeviceImpl::AddPlayoutSink(
    498     WebRtcPlayoutDataSource::Sink* sink) {
    499   DCHECK(thread_checker_.CalledOnValidThread());
    500   DCHECK(sink);
    501   base::AutoLock auto_lock(lock_);
    502   DCHECK(std::find(playout_sinks_.begin(), playout_sinks_.end(), sink) ==
    503       playout_sinks_.end());
    504   playout_sinks_.push_back(sink);
    505 }
    506 
    507 void WebRtcAudioDeviceImpl::RemovePlayoutSink(
    508     WebRtcPlayoutDataSource::Sink* sink) {
    509   DCHECK(thread_checker_.CalledOnValidThread());
    510   DCHECK(sink);
    511   base::AutoLock auto_lock(lock_);
    512   playout_sinks_.remove(sink);
    513 }
    514 
    515 bool WebRtcAudioDeviceImpl::GetAuthorizedDeviceInfoForAudioRenderer(
    516     int* session_id,
    517     int* output_sample_rate,
    518     int* output_frames_per_buffer) {
    519   DCHECK(thread_checker_.CalledOnValidThread());
    520   // If there is no capturer or there are more than one open capture devices,
    521   // return false.
    522   if (capturers_.empty() || capturers_.size() > 1)
    523     return false;
    524 
    525   return GetDefaultCapturer()->GetPairedOutputParameters(
    526       session_id, output_sample_rate, output_frames_per_buffer);
    527 }
    528 
    529 }  // namespace content
    530