Home | History | Annotate | Download | only in audio_processing 
      1 /*
      2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h"
     12 
     13 #include <assert.h>
     14 #include <string.h>
     15 
     16 #include "webrtc/modules/audio_processing/aecm/echo_control_mobile.h"
     17 #include "webrtc/modules/audio_processing/audio_buffer.h"
     18 #include "webrtc/system_wrappers/include/logging.h"
     19 
     20 namespace webrtc {
     21 
     22 typedef void Handle;
     23 
     24 namespace {
     25 int16_t MapSetting(EchoControlMobile::RoutingMode mode) {
     26   switch (mode) {
     27     case EchoControlMobile::kQuietEarpieceOrHeadset:
     28       return 0;
     29     case EchoControlMobile::kEarpiece:
     30       return 1;
     31     case EchoControlMobile::kLoudEarpiece:
     32       return 2;
     33     case EchoControlMobile::kSpeakerphone:
     34       return 3;
     35     case EchoControlMobile::kLoudSpeakerphone:
     36       return 4;
     37   }
     38   assert(false);
     39   return -1;
     40 }
     41 
     42 AudioProcessing::Error MapError(int err) {
     43   switch (err) {
     44     case AECM_UNSUPPORTED_FUNCTION_ERROR:
     45       return AudioProcessing::kUnsupportedFunctionError;
     46     case AECM_NULL_POINTER_ERROR:
     47       return AudioProcessing::kNullPointerError;
     48     case AECM_BAD_PARAMETER_ERROR:
     49       return AudioProcessing::kBadParameterError;
     50     case AECM_BAD_PARAMETER_WARNING:
     51       return AudioProcessing::kBadStreamParameterWarning;
     52     default:
     53       // AECM_UNSPECIFIED_ERROR
     54       // AECM_UNINITIALIZED_ERROR
     55       return AudioProcessing::kUnspecifiedError;
     56   }
     57 }
     58 // Maximum length that a frame of samples can have.
     59 static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160;
     60 // Maximum number of frames to buffer in the render queue.
     61 // TODO(peah): Decrease this once we properly handle hugely unbalanced
     62 // reverse and forward call numbers.
     63 static const size_t kMaxNumFramesToBuffer = 100;
     64 }  // namespace
     65 
     66 size_t EchoControlMobile::echo_path_size_bytes() {
     67     return WebRtcAecm_echo_path_size_bytes();
     68 }
     69 
     70 EchoControlMobileImpl::EchoControlMobileImpl(const AudioProcessing* apm,
     71                                              rtc::CriticalSection* crit_render,
     72                                              rtc::CriticalSection* crit_capture)
     73     : ProcessingComponent(),
     74       apm_(apm),
     75       crit_render_(crit_render),
     76       crit_capture_(crit_capture),
     77       routing_mode_(kSpeakerphone),
     78       comfort_noise_enabled_(true),
     79       external_echo_path_(NULL),
     80       render_queue_element_max_size_(0) {
     81   RTC_DCHECK(apm);
     82   RTC_DCHECK(crit_render);
     83   RTC_DCHECK(crit_capture);
     84 }
     85 
     86 EchoControlMobileImpl::~EchoControlMobileImpl() {
     87     if (external_echo_path_ != NULL) {
     88       delete [] external_echo_path_;
     89       external_echo_path_ = NULL;
     90     }
     91 }
     92 
     93 int EchoControlMobileImpl::ProcessRenderAudio(const AudioBuffer* audio) {
     94   rtc::CritScope cs_render(crit_render_);
     95 
     96   if (!is_component_enabled()) {
     97     return AudioProcessing::kNoError;
     98   }
     99 
    100   assert(audio->num_frames_per_band() <= 160);
    101   assert(audio->num_channels() == apm_->num_reverse_channels());
    102 
    103   int err = AudioProcessing::kNoError;
    104   // The ordering convention must be followed to pass to the correct AECM.
    105   size_t handle_index = 0;
    106   render_queue_buffer_.clear();
    107   for (size_t i = 0; i < apm_->num_output_channels(); i++) {
    108     for (size_t j = 0; j < audio->num_channels(); j++) {
    109       Handle* my_handle = static_cast<Handle*>(handle(handle_index));
    110       err = WebRtcAecm_GetBufferFarendError(
    111           my_handle, audio->split_bands_const(j)[kBand0To8kHz],
    112           audio->num_frames_per_band());
    113 
    114       if (err != AudioProcessing::kNoError)
    115         return MapError(err);  // TODO(ajm): warning possible?);
    116 
    117       // Buffer the samples in the render queue.
    118       render_queue_buffer_.insert(render_queue_buffer_.end(),
    119                                   audio->split_bands_const(j)[kBand0To8kHz],
    120                                   (audio->split_bands_const(j)[kBand0To8kHz] +
    121                                    audio->num_frames_per_band()));
    122 
    123       handle_index++;
    124     }
    125   }
    126 
    127   // Insert the samples into the queue.
    128   if (!render_signal_queue_->Insert(&render_queue_buffer_)) {
    129     // The data queue is full and needs to be emptied.
    130     ReadQueuedRenderData();
    131 
    132     // Retry the insert (should always work).
    133     RTC_DCHECK_EQ(render_signal_queue_->Insert(&render_queue_buffer_), true);
    134   }
    135 
    136   return AudioProcessing::kNoError;
    137 }
    138 
    139 // Read chunks of data that were received and queued on the render side from
    140 // a queue. All the data chunks are buffered into the farend signal of the AEC.
    141 void EchoControlMobileImpl::ReadQueuedRenderData() {
    142   rtc::CritScope cs_capture(crit_capture_);
    143 
    144   if (!is_component_enabled()) {
    145     return;
    146   }
    147 
    148   while (render_signal_queue_->Remove(&capture_queue_buffer_)) {
    149     size_t handle_index = 0;
    150     size_t buffer_index = 0;
    151     const size_t num_frames_per_band =
    152         capture_queue_buffer_.size() /
    153         (apm_->num_output_channels() * apm_->num_reverse_channels());
    154     for (size_t i = 0; i < apm_->num_output_channels(); i++) {
    155       for (size_t j = 0; j < apm_->num_reverse_channels(); j++) {
    156         Handle* my_handle = static_cast<Handle*>(handle(handle_index));
    157         WebRtcAecm_BufferFarend(my_handle, &capture_queue_buffer_[buffer_index],
    158                                 num_frames_per_band);
    159 
    160         buffer_index += num_frames_per_band;
    161         handle_index++;
    162       }
    163     }
    164   }
    165 }
    166 
    167 int EchoControlMobileImpl::ProcessCaptureAudio(AudioBuffer* audio) {
    168   rtc::CritScope cs_capture(crit_capture_);
    169 
    170   if (!is_component_enabled()) {
    171     return AudioProcessing::kNoError;
    172   }
    173 
    174   if (!apm_->was_stream_delay_set()) {
    175     return AudioProcessing::kStreamParameterNotSetError;
    176   }
    177 
    178   assert(audio->num_frames_per_band() <= 160);
    179   assert(audio->num_channels() == apm_->num_output_channels());
    180 
    181   int err = AudioProcessing::kNoError;
    182 
    183   // The ordering convention must be followed to pass to the correct AECM.
    184   size_t handle_index = 0;
    185   for (size_t i = 0; i < audio->num_channels(); i++) {
    186     // TODO(ajm): improve how this works, possibly inside AECM.
    187     //            This is kind of hacked up.
    188     const int16_t* noisy = audio->low_pass_reference(i);
    189     const int16_t* clean = audio->split_bands_const(i)[kBand0To8kHz];
    190     if (noisy == NULL) {
    191       noisy = clean;
    192       clean = NULL;
    193     }
    194     for (size_t j = 0; j < apm_->num_reverse_channels(); j++) {
    195       Handle* my_handle = static_cast<Handle*>(handle(handle_index));
    196       err = WebRtcAecm_Process(
    197           my_handle,
    198           noisy,
    199           clean,
    200           audio->split_bands(i)[kBand0To8kHz],
    201           audio->num_frames_per_band(),
    202           apm_->stream_delay_ms());
    203 
    204       if (err != AudioProcessing::kNoError)
    205         return MapError(err);
    206 
    207       handle_index++;
    208     }
    209   }
    210 
    211   return AudioProcessing::kNoError;
    212 }
    213 
    214 int EchoControlMobileImpl::Enable(bool enable) {
    215   // Ensure AEC and AECM are not both enabled.
    216   rtc::CritScope cs_render(crit_render_);
    217   rtc::CritScope cs_capture(crit_capture_);
    218   // The is_enabled call is safe from a deadlock perspective
    219   // as both locks are allready held in the correct order.
    220   if (enable && apm_->echo_cancellation()->is_enabled()) {
    221     return AudioProcessing::kBadParameterError;
    222   }
    223 
    224   return EnableComponent(enable);
    225 }
    226 
    227 bool EchoControlMobileImpl::is_enabled() const {
    228   rtc::CritScope cs(crit_capture_);
    229   return is_component_enabled();
    230 }
    231 
    232 int EchoControlMobileImpl::set_routing_mode(RoutingMode mode) {
    233   if (MapSetting(mode) == -1) {
    234     return AudioProcessing::kBadParameterError;
    235   }
    236 
    237   {
    238     rtc::CritScope cs(crit_capture_);
    239     routing_mode_ = mode;
    240   }
    241   return Configure();
    242 }
    243 
    244 EchoControlMobile::RoutingMode EchoControlMobileImpl::routing_mode()
    245     const {
    246   rtc::CritScope cs(crit_capture_);
    247   return routing_mode_;
    248 }
    249 
    250 int EchoControlMobileImpl::enable_comfort_noise(bool enable) {
    251   {
    252     rtc::CritScope cs(crit_capture_);
    253     comfort_noise_enabled_ = enable;
    254   }
    255   return Configure();
    256 }
    257 
    258 bool EchoControlMobileImpl::is_comfort_noise_enabled() const {
    259   rtc::CritScope cs(crit_capture_);
    260   return comfort_noise_enabled_;
    261 }
    262 
    263 int EchoControlMobileImpl::SetEchoPath(const void* echo_path,
    264                                        size_t size_bytes) {
    265   {
    266     rtc::CritScope cs_render(crit_render_);
    267     rtc::CritScope cs_capture(crit_capture_);
    268     if (echo_path == NULL) {
    269       return AudioProcessing::kNullPointerError;
    270     }
    271     if (size_bytes != echo_path_size_bytes()) {
    272       // Size mismatch
    273       return AudioProcessing::kBadParameterError;
    274     }
    275 
    276     if (external_echo_path_ == NULL) {
    277       external_echo_path_ = new unsigned char[size_bytes];
    278     }
    279     memcpy(external_echo_path_, echo_path, size_bytes);
    280   }
    281 
    282   return Initialize();
    283 }
    284 
    285 int EchoControlMobileImpl::GetEchoPath(void* echo_path,
    286                                        size_t size_bytes) const {
    287   rtc::CritScope cs(crit_capture_);
    288   if (echo_path == NULL) {
    289     return AudioProcessing::kNullPointerError;
    290   }
    291   if (size_bytes != echo_path_size_bytes()) {
    292     // Size mismatch
    293     return AudioProcessing::kBadParameterError;
    294   }
    295   if (!is_component_enabled()) {
    296     return AudioProcessing::kNotEnabledError;
    297   }
    298 
    299   // Get the echo path from the first channel
    300   Handle* my_handle = static_cast<Handle*>(handle(0));
    301   int32_t err = WebRtcAecm_GetEchoPath(my_handle, echo_path, size_bytes);
    302   if (err != 0)
    303     return MapError(err);
    304 
    305   return AudioProcessing::kNoError;
    306 }
    307 
    308 int EchoControlMobileImpl::Initialize() {
    309   {
    310     rtc::CritScope cs_capture(crit_capture_);
    311     if (!is_component_enabled()) {
    312       return AudioProcessing::kNoError;
    313     }
    314   }
    315 
    316   if (apm_->proc_sample_rate_hz() > AudioProcessing::kSampleRate16kHz) {
    317     LOG(LS_ERROR) << "AECM only supports 16 kHz or lower sample rates";
    318     return AudioProcessing::kBadSampleRateError;
    319   }
    320 
    321   int err = ProcessingComponent::Initialize();
    322   if (err != AudioProcessing::kNoError) {
    323     return err;
    324   }
    325 
    326   AllocateRenderQueue();
    327 
    328   return AudioProcessing::kNoError;
    329 }
    330 
    331 void EchoControlMobileImpl::AllocateRenderQueue() {
    332   const size_t new_render_queue_element_max_size = std::max<size_t>(
    333       static_cast<size_t>(1),
    334       kMaxAllowedValuesOfSamplesPerFrame * num_handles_required());
    335 
    336   rtc::CritScope cs_render(crit_render_);
    337   rtc::CritScope cs_capture(crit_capture_);
    338 
    339   // Reallocate the queue if the queue item size is too small to fit the
    340   // data to put in the queue.
    341   if (render_queue_element_max_size_ < new_render_queue_element_max_size) {
    342     render_queue_element_max_size_ = new_render_queue_element_max_size;
    343 
    344     std::vector<int16_t> template_queue_element(render_queue_element_max_size_);
    345 
    346     render_signal_queue_.reset(
    347         new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
    348             kMaxNumFramesToBuffer, template_queue_element,
    349             RenderQueueItemVerifier<int16_t>(render_queue_element_max_size_)));
    350 
    351     render_queue_buffer_.resize(render_queue_element_max_size_);
    352     capture_queue_buffer_.resize(render_queue_element_max_size_);
    353   } else {
    354     render_signal_queue_->Clear();
    355   }
    356 }
    357 
    358 void* EchoControlMobileImpl::CreateHandle() const {
    359   return WebRtcAecm_Create();
    360 }
    361 
    362 void EchoControlMobileImpl::DestroyHandle(void* handle) const {
    363   // This method is only called in a non-concurrent manner during APM
    364   // destruction.
    365   WebRtcAecm_Free(static_cast<Handle*>(handle));
    366 }
    367 
    368 int EchoControlMobileImpl::InitializeHandle(void* handle) const {
    369   rtc::CritScope cs_render(crit_render_);
    370   rtc::CritScope cs_capture(crit_capture_);
    371   assert(handle != NULL);
    372   Handle* my_handle = static_cast<Handle*>(handle);
    373   if (WebRtcAecm_Init(my_handle, apm_->proc_sample_rate_hz()) != 0) {
    374     return GetHandleError(my_handle);
    375   }
    376   if (external_echo_path_ != NULL) {
    377     if (WebRtcAecm_InitEchoPath(my_handle,
    378                                 external_echo_path_,
    379                                 echo_path_size_bytes()) != 0) {
    380       return GetHandleError(my_handle);
    381     }
    382   }
    383 
    384   return AudioProcessing::kNoError;
    385 }
    386 
    387 int EchoControlMobileImpl::ConfigureHandle(void* handle) const {
    388   rtc::CritScope cs_render(crit_render_);
    389   rtc::CritScope cs_capture(crit_capture_);
    390   AecmConfig config;
    391   config.cngMode = comfort_noise_enabled_;
    392   config.echoMode = MapSetting(routing_mode_);
    393 
    394   return WebRtcAecm_set_config(static_cast<Handle*>(handle), config);
    395 }
    396 
    397 size_t EchoControlMobileImpl::num_handles_required() const {
    398   // Not locked as it only relies on APM public API which is threadsafe.
    399   return apm_->num_output_channels() * apm_->num_reverse_channels();
    400 }
    401 
    402 int EchoControlMobileImpl::GetHandleError(void* handle) const {
    403   // Not locked as it does not rely on anything in the state.
    404   assert(handle != NULL);
    405   return AudioProcessing::kUnspecifiedError;
    406 }
    407 }  // namespace webrtc
    408