Home | History | Annotate | Download | only in audio
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "media/audio/audio_input_controller.h"
      6 
      7 #include "base/bind.h"
      8 #include "base/strings/stringprintf.h"
      9 #include "base/threading/thread_restrictions.h"
     10 #include "base/time/time.h"
     11 #include "media/base/limits.h"
     12 #include "media/base/scoped_histogram_timer.h"
     13 #include "media/base/user_input_monitor.h"
     14 
     15 using base::TimeDelta;
     16 
     17 namespace {
     18 const int kMaxInputChannels = 3;
     19 
     20 // TODO(henrika): remove usage of timers and add support for proper
     21 // notification of when the input device is removed.  This was originally added
     22 // to resolve http://crbug.com/79936 for Windows platforms.  This then caused
     23 // breakage (very hard to repro bugs!) on other platforms: See
     24 // http://crbug.com/226327 and http://crbug.com/230972.
     25 // See also that the timer has been disabled on Mac now due to
     26 // crbug.com/357501.
     27 const int kTimerResetIntervalSeconds = 1;
     28 // We have received reports that the timer can be too trigger happy on some
     29 // Mac devices and the initial timer interval has therefore been increased
     30 // from 1 second to 5 seconds.
     31 const int kTimerInitialIntervalSeconds = 5;
     32 
     33 #if defined(AUDIO_POWER_MONITORING)
     34 // Time constant for AudioPowerMonitor.
     35 // The utilized smoothing factor (alpha) in the exponential filter is given
     36 // by 1-exp(-1/(fs*ts)), where fs is the sample rate in Hz and ts is the time
     37 // constant given by |kPowerMeasurementTimeConstantMilliseconds|.
     38 // Example: fs=44100, ts=10e-3 => alpha~0.022420
     39 //          fs=44100, ts=20e-3 => alpha~0.165903
     40 // A large smoothing factor corresponds to a faster filter response to input
     41 // changes since y(n)=alpha*x(n)+(1-alpha)*y(n-1), where x(n) is the input
     42 // and y(n) is the output.
     43 const int kPowerMeasurementTimeConstantMilliseconds = 10;
     44 
     45 // Time in seconds between two successive measurements of audio power levels.
     46 const int kPowerMonitorLogIntervalSeconds = 5;
     47 #endif
     48 }
     49 
     50 namespace media {
     51 
     52 // static
     53 AudioInputController::Factory* AudioInputController::factory_ = NULL;
     54 
     55 AudioInputController::AudioInputController(EventHandler* handler,
     56                                            SyncWriter* sync_writer,
     57                                            UserInputMonitor* user_input_monitor)
     58     : creator_task_runner_(base::MessageLoopProxy::current()),
     59       handler_(handler),
     60       stream_(NULL),
     61       data_is_active_(false),
     62       state_(CLOSED),
     63       sync_writer_(sync_writer),
     64       max_volume_(0.0),
     65       user_input_monitor_(user_input_monitor),
     66       prev_key_down_count_(0) {
     67   DCHECK(creator_task_runner_.get());
     68 }
     69 
     70 AudioInputController::~AudioInputController() {
     71   DCHECK_EQ(state_, CLOSED);
     72 }
     73 
     74 // static
     75 scoped_refptr<AudioInputController> AudioInputController::Create(
     76     AudioManager* audio_manager,
     77     EventHandler* event_handler,
     78     const AudioParameters& params,
     79     const std::string& device_id,
     80     UserInputMonitor* user_input_monitor) {
     81   DCHECK(audio_manager);
     82 
     83   if (!params.IsValid() || (params.channels() > kMaxInputChannels))
     84     return NULL;
     85 
     86   if (factory_) {
     87     return factory_->Create(
     88         audio_manager, event_handler, params, user_input_monitor);
     89   }
     90   scoped_refptr<AudioInputController> controller(
     91       new AudioInputController(event_handler, NULL, user_input_monitor));
     92 
     93   controller->task_runner_ = audio_manager->GetTaskRunner();
     94 
     95   // Create and open a new audio input stream from the existing
     96   // audio-device thread.
     97   if (!controller->task_runner_->PostTask(FROM_HERE,
     98           base::Bind(&AudioInputController::DoCreate, controller,
     99                      base::Unretained(audio_manager), params, device_id))) {
    100     controller = NULL;
    101   }
    102 
    103   return controller;
    104 }
    105 
    106 // static
    107 scoped_refptr<AudioInputController> AudioInputController::CreateLowLatency(
    108     AudioManager* audio_manager,
    109     EventHandler* event_handler,
    110     const AudioParameters& params,
    111     const std::string& device_id,
    112     SyncWriter* sync_writer,
    113     UserInputMonitor* user_input_monitor) {
    114   DCHECK(audio_manager);
    115   DCHECK(sync_writer);
    116 
    117   if (!params.IsValid() || (params.channels() > kMaxInputChannels))
    118     return NULL;
    119 
    120   // Create the AudioInputController object and ensure that it runs on
    121   // the audio-manager thread.
    122   scoped_refptr<AudioInputController> controller(
    123       new AudioInputController(event_handler, sync_writer, user_input_monitor));
    124   controller->task_runner_ = audio_manager->GetTaskRunner();
    125 
    126   // Create and open a new audio input stream from the existing
    127   // audio-device thread. Use the provided audio-input device.
    128   if (!controller->task_runner_->PostTask(FROM_HERE,
    129           base::Bind(&AudioInputController::DoCreate, controller,
    130                      base::Unretained(audio_manager), params, device_id))) {
    131     controller = NULL;
    132   }
    133 
    134   return controller;
    135 }
    136 
    137 // static
    138 scoped_refptr<AudioInputController> AudioInputController::CreateForStream(
    139     const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
    140     EventHandler* event_handler,
    141     AudioInputStream* stream,
    142     SyncWriter* sync_writer,
    143     UserInputMonitor* user_input_monitor) {
    144   DCHECK(sync_writer);
    145   DCHECK(stream);
    146 
    147   // Create the AudioInputController object and ensure that it runs on
    148   // the audio-manager thread.
    149   scoped_refptr<AudioInputController> controller(
    150       new AudioInputController(event_handler, sync_writer, user_input_monitor));
    151   controller->task_runner_ = task_runner;
    152 
    153   // TODO(miu): See TODO at top of file.  Until that's resolved, we need to
    154   // disable the error auto-detection here (since the audio mirroring
    155   // implementation will reliably report error and close events).  Note, of
    156   // course, that we're assuming CreateForStream() has been called for the audio
    157   // mirroring use case only.
    158   if (!controller->task_runner_->PostTask(
    159           FROM_HERE,
    160           base::Bind(&AudioInputController::DoCreateForStream, controller,
    161                      stream, false))) {
    162     controller = NULL;
    163   }
    164 
    165   return controller;
    166 }
    167 
    168 void AudioInputController::Record() {
    169   task_runner_->PostTask(FROM_HERE, base::Bind(
    170       &AudioInputController::DoRecord, this));
    171 }
    172 
    173 void AudioInputController::Close(const base::Closure& closed_task) {
    174   DCHECK(!closed_task.is_null());
    175   DCHECK(creator_task_runner_->BelongsToCurrentThread());
    176 
    177   task_runner_->PostTaskAndReply(
    178       FROM_HERE, base::Bind(&AudioInputController::DoClose, this), closed_task);
    179 }
    180 
    181 void AudioInputController::SetVolume(double volume) {
    182   task_runner_->PostTask(FROM_HERE, base::Bind(
    183       &AudioInputController::DoSetVolume, this, volume));
    184 }
    185 
    186 void AudioInputController::SetAutomaticGainControl(bool enabled) {
    187   task_runner_->PostTask(FROM_HERE, base::Bind(
    188       &AudioInputController::DoSetAutomaticGainControl, this, enabled));
    189 }
    190 
    191 void AudioInputController::DoCreate(AudioManager* audio_manager,
    192                                     const AudioParameters& params,
    193                                     const std::string& device_id) {
    194   DCHECK(task_runner_->BelongsToCurrentThread());
    195   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime");
    196 
    197 #if defined(AUDIO_POWER_MONITORING)
    198   // Create the audio (power) level meter given the provided audio parameters.
    199   // An AudioBus is also needed to wrap the raw data buffer from the native
    200   // layer to match AudioPowerMonitor::Scan().
    201   // TODO(henrika): Remove use of extra AudioBus. See http://crbug.com/375155.
    202   audio_level_.reset(new media::AudioPowerMonitor(
    203       params.sample_rate(),
    204       TimeDelta::FromMilliseconds(kPowerMeasurementTimeConstantMilliseconds)));
    205   audio_params_ = params;
    206 #endif
    207 
    208   // TODO(miu): See TODO at top of file.  Until that's resolved, assume all
    209   // platform audio input requires the |no_data_timer_| be used to auto-detect
    210   // errors.  In reality, probably only Windows needs to be treated as
    211   // unreliable here.
    212   DoCreateForStream(audio_manager->MakeAudioInputStream(params, device_id),
    213                     true);
    214 }
    215 
    216 void AudioInputController::DoCreateForStream(
    217     AudioInputStream* stream_to_control, bool enable_nodata_timer) {
    218   DCHECK(task_runner_->BelongsToCurrentThread());
    219 
    220   DCHECK(!stream_);
    221   stream_ = stream_to_control;
    222 
    223   if (!stream_) {
    224     if (handler_)
    225       handler_->OnError(this, STREAM_CREATE_ERROR);
    226     return;
    227   }
    228 
    229   if (stream_ && !stream_->Open()) {
    230     stream_->Close();
    231     stream_ = NULL;
    232     if (handler_)
    233       handler_->OnError(this, STREAM_OPEN_ERROR);
    234     return;
    235   }
    236 
    237   DCHECK(!no_data_timer_.get());
    238 
    239   // The timer is enabled for logging purposes. The NO_DATA_ERROR triggered
    240   // from the timer must be ignored by the EventHandler.
    241   // TODO(henrika): remove usage of timer when it has been verified on Canary
    242   // that we are safe doing so. Goal is to get rid of |no_data_timer_| and
    243   // everything that is tied to it. crbug.com/357569.
    244   enable_nodata_timer = true;
    245 
    246   if (enable_nodata_timer) {
    247     // Create the data timer which will call FirstCheckForNoData(). The timer
    248     // is started in DoRecord() and restarted in each DoCheckForNoData()
    249     // callback.
    250     no_data_timer_.reset(new base::Timer(
    251         FROM_HERE, base::TimeDelta::FromSeconds(kTimerInitialIntervalSeconds),
    252         base::Bind(&AudioInputController::FirstCheckForNoData,
    253                    base::Unretained(this)), false));
    254   } else {
    255     DVLOG(1) << "Disabled: timer check for no data.";
    256   }
    257 
    258   state_ = CREATED;
    259   if (handler_)
    260     handler_->OnCreated(this);
    261 
    262   if (user_input_monitor_) {
    263     user_input_monitor_->EnableKeyPressMonitoring();
    264     prev_key_down_count_ = user_input_monitor_->GetKeyPressCount();
    265   }
    266 }
    267 
    268 void AudioInputController::DoRecord() {
    269   DCHECK(task_runner_->BelongsToCurrentThread());
    270   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.RecordTime");
    271 
    272   if (state_ != CREATED)
    273     return;
    274 
    275   {
    276     base::AutoLock auto_lock(lock_);
    277     state_ = RECORDING;
    278   }
    279 
    280   if (no_data_timer_) {
    281     // Start the data timer. Once |kTimerResetIntervalSeconds| have passed,
    282     // a callback to FirstCheckForNoData() is made.
    283     no_data_timer_->Reset();
    284   }
    285 
    286   stream_->Start(this);
    287   if (handler_)
    288     handler_->OnRecording(this);
    289 }
    290 
    291 void AudioInputController::DoClose() {
    292   DCHECK(task_runner_->BelongsToCurrentThread());
    293   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CloseTime");
    294 
    295   if (state_ == CLOSED)
    296     return;
    297 
    298   // Delete the timer on the same thread that created it.
    299   no_data_timer_.reset();
    300 
    301   DoStopCloseAndClearStream();
    302   SetDataIsActive(false);
    303 
    304   if (SharedMemoryAndSyncSocketMode())
    305     sync_writer_->Close();
    306 
    307   if (user_input_monitor_)
    308     user_input_monitor_->DisableKeyPressMonitoring();
    309 
    310   state_ = CLOSED;
    311 }
    312 
    313 void AudioInputController::DoReportError() {
    314   DCHECK(task_runner_->BelongsToCurrentThread());
    315   if (handler_)
    316     handler_->OnError(this, STREAM_ERROR);
    317 }
    318 
    319 void AudioInputController::DoSetVolume(double volume) {
    320   DCHECK(task_runner_->BelongsToCurrentThread());
    321   DCHECK_GE(volume, 0);
    322   DCHECK_LE(volume, 1.0);
    323 
    324   if (state_ != CREATED && state_ != RECORDING)
    325     return;
    326 
    327   // Only ask for the maximum volume at first call and use cached value
    328   // for remaining function calls.
    329   if (!max_volume_) {
    330     max_volume_ = stream_->GetMaxVolume();
    331   }
    332 
    333   if (max_volume_ == 0.0) {
    334     DLOG(WARNING) << "Failed to access input volume control";
    335     return;
    336   }
    337 
    338   // Set the stream volume and scale to a range matched to the platform.
    339   stream_->SetVolume(max_volume_ * volume);
    340 }
    341 
    342 void AudioInputController::DoSetAutomaticGainControl(bool enabled) {
    343   DCHECK(task_runner_->BelongsToCurrentThread());
    344   DCHECK_NE(state_, RECORDING);
    345 
    346   // Ensure that the AGC state only can be modified before streaming starts.
    347   if (state_ != CREATED)
    348     return;
    349 
    350   stream_->SetAutomaticGainControl(enabled);
    351 }
    352 
    353 void AudioInputController::FirstCheckForNoData() {
    354   DCHECK(task_runner_->BelongsToCurrentThread());
    355   UMA_HISTOGRAM_BOOLEAN("Media.AudioInputControllerCaptureStartupSuccess",
    356                         GetDataIsActive());
    357   DoCheckForNoData();
    358 }
    359 
    360 void AudioInputController::DoCheckForNoData() {
    361   DCHECK(task_runner_->BelongsToCurrentThread());
    362 
    363   if (!GetDataIsActive()) {
    364     // The data-is-active marker will be false only if it has been more than
    365     // one second since a data packet was recorded. This can happen if a
    366     // capture device has been removed or disabled.
    367     if (handler_)
    368       handler_->OnError(this, NO_DATA_ERROR);
    369   }
    370 
    371   // Mark data as non-active. The flag will be re-enabled in OnData() each
    372   // time a data packet is received. Hence, under normal conditions, the
    373   // flag will only be disabled during a very short period.
    374   SetDataIsActive(false);
    375 
    376   // Restart the timer to ensure that we check the flag again in
    377   // |kTimerResetIntervalSeconds|.
    378   no_data_timer_->Start(
    379       FROM_HERE, base::TimeDelta::FromSeconds(kTimerResetIntervalSeconds),
    380       base::Bind(&AudioInputController::DoCheckForNoData,
    381       base::Unretained(this)));
    382 }
    383 
    384 void AudioInputController::OnData(AudioInputStream* stream,
    385                                   const AudioBus* source,
    386                                   uint32 hardware_delay_bytes,
    387                                   double volume) {
    388   // Mark data as active to ensure that the periodic calls to
    389   // DoCheckForNoData() does not report an error to the event handler.
    390   SetDataIsActive(true);
    391 
    392   {
    393     base::AutoLock auto_lock(lock_);
    394     if (state_ != RECORDING)
    395       return;
    396   }
    397 
    398   bool key_pressed = false;
    399   if (user_input_monitor_) {
    400     size_t current_count = user_input_monitor_->GetKeyPressCount();
    401     key_pressed = current_count != prev_key_down_count_;
    402     prev_key_down_count_ = current_count;
    403     DVLOG_IF(6, key_pressed) << "Detected keypress.";
    404   }
    405 
    406   // Use SharedMemory and SyncSocket if the client has created a SyncWriter.
    407   // Used by all low-latency clients except WebSpeech.
    408   if (SharedMemoryAndSyncSocketMode()) {
    409     sync_writer_->Write(source, volume, key_pressed);
    410     sync_writer_->UpdateRecordedBytes(hardware_delay_bytes);
    411 
    412 #if defined(AUDIO_POWER_MONITORING)
    413     // Only do power-level measurements if an AudioPowerMonitor object has
    414     // been created. Done in DoCreate() but not DoCreateForStream(), hence
    415     // logging will mainly be done for WebRTC and WebSpeech clients.
    416     if (!audio_level_)
    417       return;
    418 
    419     // Perform periodic audio (power) level measurements.
    420     if ((base::TimeTicks::Now() - last_audio_level_log_time_).InSeconds() >
    421         kPowerMonitorLogIntervalSeconds) {
    422       // Wrap data into an AudioBus to match AudioPowerMonitor::Scan.
    423       // TODO(henrika): remove this section when capture side uses AudioBus.
    424       // See http://crbug.com/375155 for details.
    425       audio_level_->Scan(*source, source->frames());
    426 
    427       // Get current average power level and add it to the log.
    428       // Possible range is given by [-inf, 0] dBFS.
    429       std::pair<float, bool> result = audio_level_->ReadCurrentPowerAndClip();
    430 
    431       // Use event handler on the audio thread to relay a message to the ARIH
    432       // in content which does the actual logging on the IO thread.
    433       task_runner_->PostTask(
    434           FROM_HERE,
    435           base::Bind(
    436               &AudioInputController::DoLogAudioLevel, this, result.first));
    437 
    438       last_audio_level_log_time_ = base::TimeTicks::Now();
    439 
    440       // Reset the average power level (since we don't log continuously).
    441       audio_level_->Reset();
    442     }
    443 #endif
    444     return;
    445   }
    446 
    447   // TODO(henrika): Investigate if we can avoid the extra copy here.
    448   // (see http://crbug.com/249316 for details). AFAIK, this scope is only
    449   // active for WebSpeech clients.
    450   scoped_ptr<AudioBus> audio_data =
    451       AudioBus::Create(source->channels(), source->frames());
    452   source->CopyTo(audio_data.get());
    453 
    454   // Ownership of the audio buffer will be with the callback until it is run,
    455   // when ownership is passed to the callback function.
    456   task_runner_->PostTask(
    457       FROM_HERE,
    458       base::Bind(
    459           &AudioInputController::DoOnData, this, base::Passed(&audio_data)));
    460 }
    461 
    462 void AudioInputController::DoOnData(scoped_ptr<AudioBus> data) {
    463   DCHECK(task_runner_->BelongsToCurrentThread());
    464   if (handler_)
    465     handler_->OnData(this, data.get());
    466 }
    467 
    468 void AudioInputController::DoLogAudioLevel(float level_dbfs) {
    469 #if defined(AUDIO_POWER_MONITORING)
    470   DCHECK(task_runner_->BelongsToCurrentThread());
    471   if (!handler_)
    472     return;
    473 
    474   std::string log_string = base::StringPrintf(
    475       "AIC::OnData: average audio level=%.2f dBFS", level_dbfs);
    476   static const float kSilenceThresholdDBFS = -72.24719896f;
    477   if (level_dbfs < kSilenceThresholdDBFS)
    478     log_string += " <=> no audio input!";
    479 
    480   handler_->OnLog(this, log_string);
    481 #endif
    482 }
    483 
    484 void AudioInputController::OnError(AudioInputStream* stream) {
    485   // Handle error on the audio-manager thread.
    486   task_runner_->PostTask(FROM_HERE, base::Bind(
    487       &AudioInputController::DoReportError, this));
    488 }
    489 
    490 void AudioInputController::DoStopCloseAndClearStream() {
    491   DCHECK(task_runner_->BelongsToCurrentThread());
    492 
    493   // Allow calling unconditionally and bail if we don't have a stream to close.
    494   if (stream_ != NULL) {
    495     stream_->Stop();
    496     stream_->Close();
    497     stream_ = NULL;
    498   }
    499 
    500   // The event handler should not be touched after the stream has been closed.
    501   handler_ = NULL;
    502 }
    503 
    504 void AudioInputController::SetDataIsActive(bool enabled) {
    505   base::subtle::Release_Store(&data_is_active_, enabled);
    506 }
    507 
    508 bool AudioInputController::GetDataIsActive() {
    509   return (base::subtle::Acquire_Load(&data_is_active_) != false);
    510 }
    511 
    512 }  // namespace media
    513