Home | History | Annotate | Download | only in speech 
      1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "content/browser/speech/speech_recognizer_impl.h"
      6 
      7 #include "base/basictypes.h"
      8 #include "base/bind.h"
      9 #include "base/time/time.h"
     10 #include "content/browser/browser_main_loop.h"
     11 #include "content/browser/media/media_internals.h"
     12 #include "content/browser/speech/audio_buffer.h"
     13 #include "content/browser/speech/google_one_shot_remote_engine.h"
     14 #include "content/public/browser/speech_recognition_event_listener.h"
     15 #include "media/base/audio_converter.h"
     16 #include "net/url_request/url_request_context_getter.h"
     17 
     18 #if defined(OS_WIN)
     19 #include "media/audio/win/core_audio_util_win.h"
     20 #endif
     21 
     22 using media::AudioBus;
     23 using media::AudioConverter;
     24 using media::AudioInputController;
     25 using media::AudioManager;
     26 using media::AudioParameters;
     27 using media::ChannelLayout;
     28 
     29 namespace content {
     30 
     31 // Private class which encapsulates the audio converter and the
     32 // AudioConverter::InputCallback. It handles resampling, buffering and
     33 // channel mixing between input and output parameters.
     34 class SpeechRecognizerImpl::OnDataConverter
     35     : public media::AudioConverter::InputCallback {
     36  public:
     37   OnDataConverter(const AudioParameters& input_params,
     38                   const AudioParameters& output_params);
     39   virtual ~OnDataConverter();
     40 
     41   // Converts input audio |data| bus into an AudioChunk where the input format
     42   // is given by |input_parameters_| and the output format by
     43   // |output_parameters_|.
     44   scoped_refptr<AudioChunk> Convert(const AudioBus* data);
     45 
     46  private:
     47   // media::AudioConverter::InputCallback implementation.
     48   virtual double ProvideInput(AudioBus* dest,
     49                               base::TimeDelta buffer_delay) OVERRIDE;
     50 
     51   // Handles resampling, buffering, and channel mixing between input and output
     52   // parameters.
     53   AudioConverter audio_converter_;
     54 
     55   scoped_ptr<AudioBus> input_bus_;
     56   scoped_ptr<AudioBus> output_bus_;
     57   const AudioParameters input_parameters_;
     58   const AudioParameters output_parameters_;
     59   bool waiting_for_input_;
     60   scoped_ptr<uint8[]> converted_data_;
     61 
     62   DISALLOW_COPY_AND_ASSIGN(OnDataConverter);
     63 };
     64 
     65 namespace {
     66 
     67 // The following constants are related to the volume level indicator shown in
     68 // the UI for recorded audio.
     69 // Multiplier used when new volume is greater than previous level.
     70 const float kUpSmoothingFactor = 1.0f;
     71 // Multiplier used when new volume is lesser than previous level.
     72 const float kDownSmoothingFactor = 0.7f;
     73 // RMS dB value of a maximum (unclipped) sine wave for int16 samples.
     74 const float kAudioMeterMaxDb = 90.31f;
     75 // This value corresponds to RMS dB for int16 with 6 most-significant-bits = 0.
     76 // Values lower than this will display as empty level-meter.
     77 const float kAudioMeterMinDb = 30.0f;
     78 const float kAudioMeterDbRange = kAudioMeterMaxDb - kAudioMeterMinDb;
     79 
     80 // Maximum level to draw to display unclipped meter. (1.0f displays clipping.)
     81 const float kAudioMeterRangeMaxUnclipped = 47.0f / 48.0f;
     82 
     83 // Returns true if more than 5% of the samples are at min or max value.
     84 bool DetectClipping(const AudioChunk& chunk) {
     85   const int num_samples = chunk.NumSamples();
     86   const int16* samples = chunk.SamplesData16();
     87   const int kThreshold = num_samples / 20;
     88   int clipping_samples = 0;
     89 
     90   for (int i = 0; i < num_samples; ++i) {
     91     if (samples[i] <= -32767 || samples[i] >= 32767) {
     92       if (++clipping_samples > kThreshold)
     93         return true;
     94     }
     95   }
     96   return false;
     97 }
     98 
     99 void KeepAudioControllerRefcountedForDtor(scoped_refptr<AudioInputController>) {
    100 }
    101 
    102 }  // namespace
    103 
    104 const int SpeechRecognizerImpl::kAudioSampleRate = 16000;
    105 const ChannelLayout SpeechRecognizerImpl::kChannelLayout =
    106     media::CHANNEL_LAYOUT_MONO;
    107 const int SpeechRecognizerImpl::kNumBitsPerAudioSample = 16;
    108 const int SpeechRecognizerImpl::kNoSpeechTimeoutMs = 8000;
    109 const int SpeechRecognizerImpl::kEndpointerEstimationTimeMs = 300;
    110 media::AudioManager* SpeechRecognizerImpl::audio_manager_for_tests_ = NULL;
    111 
    112 COMPILE_ASSERT(SpeechRecognizerImpl::kNumBitsPerAudioSample % 8 == 0,
    113                kNumBitsPerAudioSample_must_be_a_multiple_of_8);
    114 
    115 // SpeechRecognizerImpl::OnDataConverter implementation
    116 
    117 SpeechRecognizerImpl::OnDataConverter::OnDataConverter(
    118     const AudioParameters& input_params, const AudioParameters& output_params)
    119     : audio_converter_(input_params, output_params, false),
    120       input_bus_(AudioBus::Create(input_params)),
    121       output_bus_(AudioBus::Create(output_params)),
    122       input_parameters_(input_params),
    123       output_parameters_(output_params),
    124       waiting_for_input_(false),
    125       converted_data_(new uint8[output_parameters_.GetBytesPerBuffer()]) {
    126   audio_converter_.AddInput(this);
    127 }
    128 
    129 SpeechRecognizerImpl::OnDataConverter::~OnDataConverter() {
    130   // It should now be safe to unregister the converter since no more OnData()
    131   // callbacks are outstanding at this point.
    132   audio_converter_.RemoveInput(this);
    133 }
    134 
    135 scoped_refptr<AudioChunk> SpeechRecognizerImpl::OnDataConverter::Convert(
    136     const AudioBus* data) {
    137   CHECK_EQ(data->frames(), input_parameters_.frames_per_buffer());
    138 
    139   data->CopyTo(input_bus_.get());
    140 
    141   waiting_for_input_ = true;
    142   audio_converter_.Convert(output_bus_.get());
    143 
    144   output_bus_->ToInterleaved(
    145       output_bus_->frames(), output_parameters_.bits_per_sample() / 8,
    146       converted_data_.get());
    147 
    148   // TODO(primiano): Refactor AudioChunk to avoid the extra-copy here
    149   // (see http://crbug.com/249316 for details).
    150   return scoped_refptr<AudioChunk>(new AudioChunk(
    151       converted_data_.get(),
    152       output_parameters_.GetBytesPerBuffer(),
    153       output_parameters_.bits_per_sample() / 8));
    154 }
    155 
    156 double SpeechRecognizerImpl::OnDataConverter::ProvideInput(
    157     AudioBus* dest, base::TimeDelta buffer_delay) {
    158   // The audio converted should never ask for more than one bus in each call
    159   // to Convert(). If so, we have a serious issue in our design since we might
    160   // miss recorded chunks of 100 ms audio data.
    161   CHECK(waiting_for_input_);
    162 
    163   // Read from the input bus to feed the converter.
    164   input_bus_->CopyTo(dest);
    165 
    166   // |input_bus_| should only be provide once.
    167   waiting_for_input_ = false;
    168   return 1;
    169 }
    170 
    171 // SpeechRecognizerImpl implementation
    172 
    173 SpeechRecognizerImpl::SpeechRecognizerImpl(
    174     SpeechRecognitionEventListener* listener,
    175     int session_id,
    176     bool continuous,
    177     bool provisional_results,
    178     SpeechRecognitionEngine* engine)
    179     : SpeechRecognizer(listener, session_id),
    180       recognition_engine_(engine),
    181       endpointer_(kAudioSampleRate),
    182       audio_log_(MediaInternals::GetInstance()->CreateAudioLog(
    183           media::AudioLogFactory::AUDIO_INPUT_CONTROLLER)),
    184       is_dispatching_event_(false),
    185       provisional_results_(provisional_results),
    186       state_(STATE_IDLE) {
    187   DCHECK(recognition_engine_ != NULL);
    188   if (!continuous) {
    189     // In single shot (non-continous) recognition,
    190     // the session is automatically ended after:
    191     //  - 0.5 seconds of silence if time <  3 seconds
    192     //  - 1   seconds of silence if time >= 3 seconds
    193     endpointer_.set_speech_input_complete_silence_length(
    194         base::Time::kMicrosecondsPerSecond / 2);
    195     endpointer_.set_long_speech_input_complete_silence_length(
    196         base::Time::kMicrosecondsPerSecond);
    197     endpointer_.set_long_speech_length(3 * base::Time::kMicrosecondsPerSecond);
    198   } else {
    199     // In continuous recognition, the session is automatically ended after 15
    200     // seconds of silence.
    201     const int64 cont_timeout_us = base::Time::kMicrosecondsPerSecond * 15;
    202     endpointer_.set_speech_input_complete_silence_length(cont_timeout_us);
    203     endpointer_.set_long_speech_length(0);  // Use only a single timeout.
    204   }
    205   endpointer_.StartSession();
    206   recognition_engine_->set_delegate(this);
    207 }
    208 
    209 // -------  Methods that trigger Finite State Machine (FSM) events ------------
    210 
    211 // NOTE:all the external events and requests should be enqueued (PostTask), even
    212 // if they come from the same (IO) thread, in order to preserve the relationship
    213 // of causality between events and avoid interleaved event processing due to
    214 // synchronous callbacks.
    215 
    216 void SpeechRecognizerImpl::StartRecognition(const std::string& device_id) {
    217   DCHECK(!device_id.empty());
    218   device_id_ = device_id;
    219 
    220   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
    221                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
    222                                      this, FSMEventArgs(EVENT_START)));
    223 }
    224 
    225 void SpeechRecognizerImpl::AbortRecognition() {
    226   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
    227                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
    228                                      this, FSMEventArgs(EVENT_ABORT)));
    229 }
    230 
    231 void SpeechRecognizerImpl::StopAudioCapture() {
    232   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
    233                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
    234                                      this, FSMEventArgs(EVENT_STOP_CAPTURE)));
    235 }
    236 
    237 bool SpeechRecognizerImpl::IsActive() const {
    238   // Checking the FSM state from another thread (thus, while the FSM is
    239   // potentially concurrently evolving) is meaningless.
    240   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    241   return state_ != STATE_IDLE && state_ != STATE_ENDED;
    242 }
    243 
    244 bool SpeechRecognizerImpl::IsCapturingAudio() const {
    245   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); // See IsActive().
    246   const bool is_capturing_audio = state_ >= STATE_STARTING &&
    247                                   state_ <= STATE_RECOGNIZING;
    248   DCHECK((is_capturing_audio && (audio_controller_.get() != NULL)) ||
    249          (!is_capturing_audio && audio_controller_.get() == NULL));
    250   return is_capturing_audio;
    251 }
    252 
    253 const SpeechRecognitionEngine&
    254 SpeechRecognizerImpl::recognition_engine() const {
    255   return *(recognition_engine_.get());
    256 }
    257 
    258 SpeechRecognizerImpl::~SpeechRecognizerImpl() {
    259   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    260   endpointer_.EndSession();
    261   if (audio_controller_.get()) {
    262     audio_controller_->Close(
    263         base::Bind(&KeepAudioControllerRefcountedForDtor, audio_controller_));
    264     audio_log_->OnClosed(0);
    265   }
    266 }
    267 
    268 // Invoked in the audio thread.
    269 void SpeechRecognizerImpl::OnError(AudioInputController* controller,
    270     media::AudioInputController::ErrorCode error_code) {
    271   FSMEventArgs event_args(EVENT_AUDIO_ERROR);
    272   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
    273                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
    274                                      this, event_args));
    275 }
    276 
    277 void SpeechRecognizerImpl::OnData(AudioInputController* controller,
    278                                   const AudioBus* data) {
    279   // Convert audio from native format to fixed format used by WebSpeech.
    280   FSMEventArgs event_args(EVENT_AUDIO_DATA);
    281   event_args.audio_data = audio_converter_->Convert(data);
    282 
    283   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
    284                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
    285                                      this, event_args));
    286 }
    287 
    288 void SpeechRecognizerImpl::OnAudioClosed(AudioInputController*) {}
    289 
    290 void SpeechRecognizerImpl::OnSpeechRecognitionEngineResults(
    291     const SpeechRecognitionResults& results) {
    292   FSMEventArgs event_args(EVENT_ENGINE_RESULT);
    293   event_args.engine_results = results;
    294   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
    295                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
    296                                      this, event_args));
    297 }
    298 
    299 void SpeechRecognizerImpl::OnSpeechRecognitionEngineError(
    300     const SpeechRecognitionError& error) {
    301   FSMEventArgs event_args(EVENT_ENGINE_ERROR);
    302   event_args.engine_error = error;
    303   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
    304                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
    305                                      this, event_args));
    306 }
    307 
    308 // -----------------------  Core FSM implementation ---------------------------
    309 // TODO(primiano): After the changes in the media package (r129173), this class
    310 // slightly violates the SpeechRecognitionEventListener interface contract. In
    311 // particular, it is not true anymore that this class can be freed after the
    312 // OnRecognitionEnd event, since the audio_controller_.Close() asynchronous
    313 // call can be still in progress after the end event. Currently, it does not
    314 // represent a problem for the browser itself, since refcounting protects us
    315 // against such race conditions. However, we should fix this in the next CLs.
    316 // For instance, tests are currently working just because the
    317 // TestAudioInputController is not closing asynchronously as the real controller
    318 // does, but they will become flaky if TestAudioInputController will be fixed.
    319 
    320 void SpeechRecognizerImpl::DispatchEvent(const FSMEventArgs& event_args) {
    321   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    322   DCHECK_LE(event_args.event, EVENT_MAX_VALUE);
    323   DCHECK_LE(state_, STATE_MAX_VALUE);
    324 
    325   // Event dispatching must be sequential, otherwise it will break all the rules
    326   // and the assumptions of the finite state automata model.
    327   DCHECK(!is_dispatching_event_);
    328   is_dispatching_event_ = true;
    329 
    330   // Guard against the delegate freeing us until we finish processing the event.
    331   scoped_refptr<SpeechRecognizerImpl> me(this);
    332 
    333   if (event_args.event == EVENT_AUDIO_DATA) {
    334     DCHECK(event_args.audio_data.get() != NULL);
    335     ProcessAudioPipeline(*event_args.audio_data.get());
    336   }
    337 
    338   // The audio pipeline must be processed before the event dispatch, otherwise
    339   // it would take actions according to the future state instead of the current.
    340   state_ = ExecuteTransitionAndGetNextState(event_args);
    341   is_dispatching_event_ = false;
    342 }
    343 
    344 SpeechRecognizerImpl::FSMState
    345 SpeechRecognizerImpl::ExecuteTransitionAndGetNextState(
    346     const FSMEventArgs& event_args) {
    347   const FSMEvent event = event_args.event;
    348   switch (state_) {
    349     case STATE_IDLE:
    350       switch (event) {
    351         // TODO(primiano): restore UNREACHABLE_CONDITION on EVENT_ABORT and
    352         // EVENT_STOP_CAPTURE below once speech input extensions are fixed.
    353         case EVENT_ABORT:
    354           return AbortSilently(event_args);
    355         case EVENT_START:
    356           return StartRecording(event_args);
    357         case EVENT_STOP_CAPTURE:
    358           return AbortSilently(event_args);
    359         case EVENT_AUDIO_DATA:     // Corner cases related to queued messages
    360         case EVENT_ENGINE_RESULT:  // being lately dispatched.
    361         case EVENT_ENGINE_ERROR:
    362         case EVENT_AUDIO_ERROR:
    363           return DoNothing(event_args);
    364       }
    365       break;
    366     case STATE_STARTING:
    367       switch (event) {
    368         case EVENT_ABORT:
    369           return AbortWithError(event_args);
    370         case EVENT_START:
    371           return NotFeasible(event_args);
    372         case EVENT_STOP_CAPTURE:
    373           return AbortSilently(event_args);
    374         case EVENT_AUDIO_DATA:
    375           return StartRecognitionEngine(event_args);
    376         case EVENT_ENGINE_RESULT:
    377           return NotFeasible(event_args);
    378         case EVENT_ENGINE_ERROR:
    379         case EVENT_AUDIO_ERROR:
    380           return AbortWithError(event_args);
    381       }
    382       break;
    383     case STATE_ESTIMATING_ENVIRONMENT:
    384       switch (event) {
    385         case EVENT_ABORT:
    386           return AbortWithError(event_args);
    387         case EVENT_START:
    388           return NotFeasible(event_args);
    389         case EVENT_STOP_CAPTURE:
    390           return StopCaptureAndWaitForResult(event_args);
    391         case EVENT_AUDIO_DATA:
    392           return WaitEnvironmentEstimationCompletion(event_args);
    393         case EVENT_ENGINE_RESULT:
    394           return ProcessIntermediateResult(event_args);
    395         case EVENT_ENGINE_ERROR:
    396         case EVENT_AUDIO_ERROR:
    397           return AbortWithError(event_args);
    398       }
    399       break;
    400     case STATE_WAITING_FOR_SPEECH:
    401       switch (event) {
    402         case EVENT_ABORT:
    403           return AbortWithError(event_args);
    404         case EVENT_START:
    405           return NotFeasible(event_args);
    406         case EVENT_STOP_CAPTURE:
    407           return StopCaptureAndWaitForResult(event_args);
    408         case EVENT_AUDIO_DATA:
    409           return DetectUserSpeechOrTimeout(event_args);
    410         case EVENT_ENGINE_RESULT:
    411           return ProcessIntermediateResult(event_args);
    412         case EVENT_ENGINE_ERROR:
    413         case EVENT_AUDIO_ERROR:
    414           return AbortWithError(event_args);
    415       }
    416       break;
    417     case STATE_RECOGNIZING:
    418       switch (event) {
    419         case EVENT_ABORT:
    420           return AbortWithError(event_args);
    421         case EVENT_START:
    422           return NotFeasible(event_args);
    423         case EVENT_STOP_CAPTURE:
    424           return StopCaptureAndWaitForResult(event_args);
    425         case EVENT_AUDIO_DATA:
    426           return DetectEndOfSpeech(event_args);
    427         case EVENT_ENGINE_RESULT:
    428           return ProcessIntermediateResult(event_args);
    429         case EVENT_ENGINE_ERROR:
    430         case EVENT_AUDIO_ERROR:
    431           return AbortWithError(event_args);
    432       }
    433       break;
    434     case STATE_WAITING_FINAL_RESULT:
    435       switch (event) {
    436         case EVENT_ABORT:
    437           return AbortWithError(event_args);
    438         case EVENT_START:
    439           return NotFeasible(event_args);
    440         case EVENT_STOP_CAPTURE:
    441         case EVENT_AUDIO_DATA:
    442           return DoNothing(event_args);
    443         case EVENT_ENGINE_RESULT:
    444           return ProcessFinalResult(event_args);
    445         case EVENT_ENGINE_ERROR:
    446         case EVENT_AUDIO_ERROR:
    447           return AbortWithError(event_args);
    448       }
    449       break;
    450 
    451     // TODO(primiano): remove this state when speech input extensions support
    452     // will be removed and STATE_IDLE.EVENT_ABORT,EVENT_STOP_CAPTURE will be
    453     // reset to NotFeasible (see TODO above).
    454     case STATE_ENDED:
    455       return DoNothing(event_args);
    456   }
    457   return NotFeasible(event_args);
    458 }
    459 
    460 // ----------- Contract for all the FSM evolution functions below -------------
    461 //  - Are guaranteed to be executed in the IO thread;
    462 //  - Are guaranteed to be not reentrant (themselves and each other);
    463 //  - event_args members are guaranteed to be stable during the call;
    464 //  - The class won't be freed in the meanwhile due to callbacks;
    465 //  - IsCapturingAudio() returns true if and only if audio_controller_ != NULL.
    466 
    467 // TODO(primiano): the audio pipeline is currently serial. However, the
    468 // clipper->endpointer->vumeter chain and the sr_engine could be parallelized.
    469 // We should profile the execution to see if it would be worth or not.
    470 void SpeechRecognizerImpl::ProcessAudioPipeline(const AudioChunk& raw_audio) {
    471   const bool route_to_endpointer = state_ >= STATE_ESTIMATING_ENVIRONMENT &&
    472                                    state_ <= STATE_RECOGNIZING;
    473   const bool route_to_sr_engine = route_to_endpointer;
    474   const bool route_to_vumeter = state_ >= STATE_WAITING_FOR_SPEECH &&
    475                                 state_ <= STATE_RECOGNIZING;
    476   const bool clip_detected = DetectClipping(raw_audio);
    477   float rms = 0.0f;
    478 
    479   num_samples_recorded_ += raw_audio.NumSamples();
    480 
    481   if (route_to_endpointer)
    482     endpointer_.ProcessAudio(raw_audio, &rms);
    483 
    484   if (route_to_vumeter) {
    485     DCHECK(route_to_endpointer); // Depends on endpointer due to |rms|.
    486     UpdateSignalAndNoiseLevels(rms, clip_detected);
    487   }
    488   if (route_to_sr_engine) {
    489     DCHECK(recognition_engine_.get() != NULL);
    490     recognition_engine_->TakeAudioChunk(raw_audio);
    491   }
    492 }
    493 
    494 SpeechRecognizerImpl::FSMState
    495 SpeechRecognizerImpl::StartRecording(const FSMEventArgs&) {
    496   DCHECK(recognition_engine_.get() != NULL);
    497   DCHECK(!IsCapturingAudio());
    498   const bool unit_test_is_active = (audio_manager_for_tests_ != NULL);
    499   AudioManager* audio_manager = unit_test_is_active ?
    500                                 audio_manager_for_tests_ :
    501                                 AudioManager::Get();
    502   DCHECK(audio_manager != NULL);
    503 
    504   DVLOG(1) << "SpeechRecognizerImpl starting audio capture.";
    505   num_samples_recorded_ = 0;
    506   audio_level_ = 0;
    507   listener()->OnRecognitionStart(session_id());
    508 
    509   // TODO(xians): Check if the OS has the device with |device_id_|, return
    510   // |SPEECH_AUDIO_ERROR_DETAILS_NO_MIC| if the target device does not exist.
    511   if (!audio_manager->HasAudioInputDevices()) {
    512     return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO,
    513                                         SPEECH_AUDIO_ERROR_DETAILS_NO_MIC));
    514   }
    515 
    516   int chunk_duration_ms = recognition_engine_->GetDesiredAudioChunkDurationMs();
    517 
    518   AudioParameters in_params = audio_manager->GetInputStreamParameters(
    519       device_id_);
    520   if (!in_params.IsValid() && !unit_test_is_active) {
    521     DLOG(ERROR) << "Invalid native audio input parameters";
    522     return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO));
    523   }
    524 
    525   // Audio converter shall provide audio based on these parameters as output.
    526   // Hard coded, WebSpeech specific parameters are utilized here.
    527   int frames_per_buffer = (kAudioSampleRate * chunk_duration_ms) / 1000;
    528   AudioParameters output_parameters = AudioParameters(
    529       AudioParameters::AUDIO_PCM_LOW_LATENCY, kChannelLayout, kAudioSampleRate,
    530       kNumBitsPerAudioSample, frames_per_buffer);
    531 
    532   // Audio converter will receive audio based on these parameters as input.
    533   // On Windows we start by verifying that Core Audio is supported. If not,
    534   // the WaveIn API is used and we might as well avoid all audio conversations
    535   // since WaveIn does the conversion for us.
    536   // TODO(henrika): this code should be moved to platform dependent audio
    537   // managers.
    538   bool use_native_audio_params = true;
    539 #if defined(OS_WIN)
    540   use_native_audio_params = media::CoreAudioUtil::IsSupported();
    541   DVLOG_IF(1, !use_native_audio_params) << "Reverting to WaveIn for WebSpeech";
    542 #endif
    543 
    544   AudioParameters input_parameters = output_parameters;
    545   if (use_native_audio_params && !unit_test_is_active) {
    546     // Use native audio parameters but avoid opening up at the native buffer
    547     // size. Instead use same frame size (in milliseconds) as WebSpeech uses.
    548     // We rely on internal buffers in the audio back-end to fulfill this request
    549     // and the idea is to simplify the audio conversion since each Convert()
    550     // call will then render exactly one ProvideInput() call.
    551     // Due to implementation details in the audio converter, 2 milliseconds
    552     // are added to the default frame size (100 ms) to ensure there is enough
    553     // data to generate 100 ms of output when resampling.
    554     frames_per_buffer =
    555         ((in_params.sample_rate() * (chunk_duration_ms + 2)) / 1000.0) + 0.5;
    556     input_parameters.Reset(in_params.format(),
    557                            in_params.channel_layout(),
    558                            in_params.channels(),
    559                            in_params.sample_rate(),
    560                            in_params.bits_per_sample(),
    561                            frames_per_buffer);
    562   }
    563 
    564   // Create an audio converter which converts data between native input format
    565   // and WebSpeech specific output format.
    566   audio_converter_.reset(
    567       new OnDataConverter(input_parameters, output_parameters));
    568 
    569   audio_controller_ = AudioInputController::Create(
    570       audio_manager, this, input_parameters, device_id_, NULL);
    571 
    572   if (!audio_controller_.get()) {
    573     return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO));
    574   }
    575 
    576   audio_log_->OnCreated(0, input_parameters, device_id_);
    577 
    578   // The endpointer needs to estimate the environment/background noise before
    579   // starting to treat the audio as user input. We wait in the state
    580   // ESTIMATING_ENVIRONMENT until such interval has elapsed before switching
    581   // to user input mode.
    582   endpointer_.SetEnvironmentEstimationMode();
    583   audio_controller_->Record();
    584   audio_log_->OnStarted(0);
    585   return STATE_STARTING;
    586 }
    587 
    588 SpeechRecognizerImpl::FSMState
    589 SpeechRecognizerImpl::StartRecognitionEngine(const FSMEventArgs& event_args) {
    590   // This is the first audio packet captured, so the recognition engine is
    591   // started and the delegate notified about the event.
    592   DCHECK(recognition_engine_.get() != NULL);
    593   recognition_engine_->StartRecognition();
    594   listener()->OnAudioStart(session_id());
    595 
    596   // This is a little hack, since TakeAudioChunk() is already called by
    597   // ProcessAudioPipeline(). It is the best tradeoff, unless we allow dropping
    598   // the first audio chunk captured after opening the audio device.
    599   recognition_engine_->TakeAudioChunk(*(event_args.audio_data.get()));
    600   return STATE_ESTIMATING_ENVIRONMENT;
    601 }
    602 
    603 SpeechRecognizerImpl::FSMState
    604 SpeechRecognizerImpl::WaitEnvironmentEstimationCompletion(const FSMEventArgs&) {
    605   DCHECK(endpointer_.IsEstimatingEnvironment());
    606   if (GetElapsedTimeMs() >= kEndpointerEstimationTimeMs) {
    607     endpointer_.SetUserInputMode();
    608     listener()->OnEnvironmentEstimationComplete(session_id());
    609     return STATE_WAITING_FOR_SPEECH;
    610   } else {
    611     return STATE_ESTIMATING_ENVIRONMENT;
    612   }
    613 }
    614 
    615 SpeechRecognizerImpl::FSMState
    616 SpeechRecognizerImpl::DetectUserSpeechOrTimeout(const FSMEventArgs&) {
    617   if (endpointer_.DidStartReceivingSpeech()) {
    618     listener()->OnSoundStart(session_id());
    619     return STATE_RECOGNIZING;
    620   } else if (GetElapsedTimeMs() >= kNoSpeechTimeoutMs) {
    621     return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_NO_SPEECH));
    622   }
    623   return STATE_WAITING_FOR_SPEECH;
    624 }
    625 
    626 SpeechRecognizerImpl::FSMState
    627 SpeechRecognizerImpl::DetectEndOfSpeech(const FSMEventArgs& event_args) {
    628   if (endpointer_.speech_input_complete())
    629     return StopCaptureAndWaitForResult(event_args);
    630   return STATE_RECOGNIZING;
    631 }
    632 
    633 SpeechRecognizerImpl::FSMState
    634 SpeechRecognizerImpl::StopCaptureAndWaitForResult(const FSMEventArgs&) {
    635   DCHECK(state_ >= STATE_ESTIMATING_ENVIRONMENT && state_ <= STATE_RECOGNIZING);
    636 
    637   DVLOG(1) << "Concluding recognition";
    638   CloseAudioControllerAsynchronously();
    639   recognition_engine_->AudioChunksEnded();
    640 
    641   if (state_ > STATE_WAITING_FOR_SPEECH)
    642     listener()->OnSoundEnd(session_id());
    643 
    644   listener()->OnAudioEnd(session_id());
    645   return STATE_WAITING_FINAL_RESULT;
    646 }
    647 
    648 SpeechRecognizerImpl::FSMState
    649 SpeechRecognizerImpl::AbortSilently(const FSMEventArgs& event_args) {
    650   DCHECK_NE(event_args.event, EVENT_AUDIO_ERROR);
    651   DCHECK_NE(event_args.event, EVENT_ENGINE_ERROR);
    652   return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_NONE));
    653 }
    654 
    655 SpeechRecognizerImpl::FSMState
    656 SpeechRecognizerImpl::AbortWithError(const FSMEventArgs& event_args) {
    657   if (event_args.event == EVENT_AUDIO_ERROR) {
    658     return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO));
    659   } else if (event_args.event == EVENT_ENGINE_ERROR) {
    660     return Abort(event_args.engine_error);
    661   }
    662   return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_ABORTED));
    663 }
    664 
    665 SpeechRecognizerImpl::FSMState SpeechRecognizerImpl::Abort(
    666     const SpeechRecognitionError& error) {
    667   if (IsCapturingAudio())
    668     CloseAudioControllerAsynchronously();
    669 
    670   DVLOG(1) << "SpeechRecognizerImpl canceling recognition. ";
    671 
    672   // The recognition engine is initialized only after STATE_STARTING.
    673   if (state_ > STATE_STARTING) {
    674     DCHECK(recognition_engine_.get() != NULL);
    675     recognition_engine_->EndRecognition();
    676   }
    677 
    678   if (state_ > STATE_WAITING_FOR_SPEECH && state_ < STATE_WAITING_FINAL_RESULT)
    679     listener()->OnSoundEnd(session_id());
    680 
    681   if (state_ > STATE_STARTING && state_ < STATE_WAITING_FINAL_RESULT)
    682     listener()->OnAudioEnd(session_id());
    683 
    684   if (error.code != SPEECH_RECOGNITION_ERROR_NONE)
    685     listener()->OnRecognitionError(session_id(), error);
    686 
    687   listener()->OnRecognitionEnd(session_id());
    688 
    689   return STATE_ENDED;
    690 }
    691 
    692 SpeechRecognizerImpl::FSMState SpeechRecognizerImpl::ProcessIntermediateResult(
    693     const FSMEventArgs& event_args) {
    694   // Provisional results can occur only if explicitly enabled in the JS API.
    695   DCHECK(provisional_results_);
    696 
    697   // In continuous recognition, intermediate results can occur even when we are
    698   // in the ESTIMATING_ENVIRONMENT or WAITING_FOR_SPEECH states (if the
    699   // recognition engine is "faster" than our endpointer). In these cases we
    700   // skip the endpointer and fast-forward to the RECOGNIZING state, with respect
    701   // of the events triggering order.
    702   if (state_ == STATE_ESTIMATING_ENVIRONMENT) {
    703     DCHECK(endpointer_.IsEstimatingEnvironment());
    704     endpointer_.SetUserInputMode();
    705     listener()->OnEnvironmentEstimationComplete(session_id());
    706   } else if (state_ == STATE_WAITING_FOR_SPEECH) {
    707     listener()->OnSoundStart(session_id());
    708   } else {
    709     DCHECK_EQ(STATE_RECOGNIZING, state_);
    710   }
    711 
    712   listener()->OnRecognitionResults(session_id(), event_args.engine_results);
    713   return STATE_RECOGNIZING;
    714 }
    715 
    716 SpeechRecognizerImpl::FSMState
    717 SpeechRecognizerImpl::ProcessFinalResult(const FSMEventArgs& event_args) {
    718   const SpeechRecognitionResults& results = event_args.engine_results;
    719   SpeechRecognitionResults::const_iterator i = results.begin();
    720   bool provisional_results_pending = false;
    721   bool results_are_empty = true;
    722   for (; i != results.end(); ++i) {
    723     const SpeechRecognitionResult& result = *i;
    724     if (result.is_provisional) {
    725       DCHECK(provisional_results_);
    726       provisional_results_pending = true;
    727     } else if (results_are_empty) {
    728       results_are_empty = result.hypotheses.empty();
    729     }
    730   }
    731 
    732   if (provisional_results_pending) {
    733     listener()->OnRecognitionResults(session_id(), results);
    734     // We don't end the recognition if a provisional result is received in
    735     // STATE_WAITING_FINAL_RESULT. A definitive result will come next and will
    736     // end the recognition.
    737     return state_;
    738   }
    739 
    740   recognition_engine_->EndRecognition();
    741 
    742   if (!results_are_empty) {
    743     // We could receive an empty result (which we won't propagate further)
    744     // in the following (continuous) scenario:
    745     //  1. The caller start pushing audio and receives some results;
    746     //  2. A |StopAudioCapture| is issued later;
    747     //  3. The final audio frames captured in the interval ]1,2] do not lead to
    748     //     any result (nor any error);
    749     //  4. The speech recognition engine, therefore, emits an empty result to
    750     //     notify that the recognition is ended with no error, yet neither any
    751     //     further result.
    752     listener()->OnRecognitionResults(session_id(), results);
    753   }
    754 
    755   listener()->OnRecognitionEnd(session_id());
    756   return STATE_ENDED;
    757 }
    758 
    759 SpeechRecognizerImpl::FSMState
    760 SpeechRecognizerImpl::DoNothing(const FSMEventArgs&) const {
    761   return state_;  // Just keep the current state.
    762 }
    763 
    764 SpeechRecognizerImpl::FSMState
    765 SpeechRecognizerImpl::NotFeasible(const FSMEventArgs& event_args) {
    766   NOTREACHED() << "Unfeasible event " << event_args.event
    767                << " in state " << state_;
    768   return state_;
    769 }
    770 
    771 void SpeechRecognizerImpl::CloseAudioControllerAsynchronously() {
    772   DCHECK(IsCapturingAudio());
    773   DVLOG(1) << "SpeechRecognizerImpl closing audio controller.";
    774   // Issues a Close on the audio controller, passing an empty callback. The only
    775   // purpose of such callback is to keep the audio controller refcounted until
    776   // Close has completed (in the audio thread) and automatically destroy it
    777   // afterwards (upon return from OnAudioClosed).
    778   audio_controller_->Close(base::Bind(&SpeechRecognizerImpl::OnAudioClosed,
    779                                       this, audio_controller_));
    780   audio_controller_ = NULL;  // The controller is still refcounted by Bind.
    781   audio_log_->OnClosed(0);
    782 }
    783 
    784 int SpeechRecognizerImpl::GetElapsedTimeMs() const {
    785   return (num_samples_recorded_ * 1000) / kAudioSampleRate;
    786 }
    787 
    788 void SpeechRecognizerImpl::UpdateSignalAndNoiseLevels(const float& rms,
    789                                                   bool clip_detected) {
    790   // Calculate the input volume to display in the UI, smoothing towards the
    791   // new level.
    792   // TODO(primiano): Do we really need all this floating point arith here?
    793   // Perhaps it might be quite expensive on mobile.
    794   float level = (rms - kAudioMeterMinDb) /
    795       (kAudioMeterDbRange / kAudioMeterRangeMaxUnclipped);
    796   level = std::min(std::max(0.0f, level), kAudioMeterRangeMaxUnclipped);
    797   const float smoothing_factor = (level > audio_level_) ? kUpSmoothingFactor :
    798                                                           kDownSmoothingFactor;
    799   audio_level_ += (level - audio_level_) * smoothing_factor;
    800 
    801   float noise_level = (endpointer_.NoiseLevelDb() - kAudioMeterMinDb) /
    802       (kAudioMeterDbRange / kAudioMeterRangeMaxUnclipped);
    803   noise_level = std::min(std::max(0.0f, noise_level),
    804                          kAudioMeterRangeMaxUnclipped);
    805 
    806   listener()->OnAudioLevelsChange(
    807       session_id(), clip_detected ? 1.0f : audio_level_, noise_level);
    808 }
    809 
    810 void SpeechRecognizerImpl::SetAudioManagerForTesting(
    811     AudioManager* audio_manager) {
    812   audio_manager_for_tests_ = audio_manager;
    813 }
    814 
    815 SpeechRecognizerImpl::FSMEventArgs::FSMEventArgs(FSMEvent event_value)
    816     : event(event_value),
    817       audio_data(NULL),
    818       engine_error(SPEECH_RECOGNITION_ERROR_NONE) {
    819 }
    820 
    821 SpeechRecognizerImpl::FSMEventArgs::~FSMEventArgs() {
    822 }
    823 
    824 }  // namespace content
    825