Home | History | Annotate | Download | only in audio_processing 
      1 /*
      2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/modules/audio_processing/echo_cancellation_impl.h"
     12 
     13 #include <assert.h>
     14 #include <string.h>
     15 
     16 extern "C" {
     17 #include "webrtc/modules/audio_processing/aec/aec_core.h"
     18 }
     19 #include "webrtc/modules/audio_processing/aec/echo_cancellation.h"
     20 #include "webrtc/modules/audio_processing/audio_buffer.h"
     21 
     22 namespace webrtc {
     23 
     24 typedef void Handle;
     25 
     26 namespace {
     27 int16_t MapSetting(EchoCancellation::SuppressionLevel level) {
     28   switch (level) {
     29     case EchoCancellation::kLowSuppression:
     30       return kAecNlpConservative;
     31     case EchoCancellation::kModerateSuppression:
     32       return kAecNlpModerate;
     33     case EchoCancellation::kHighSuppression:
     34       return kAecNlpAggressive;
     35   }
     36   assert(false);
     37   return -1;
     38 }
     39 
     40 AudioProcessing::Error MapError(int err) {
     41   switch (err) {
     42     case AEC_UNSUPPORTED_FUNCTION_ERROR:
     43       return AudioProcessing::kUnsupportedFunctionError;
     44     case AEC_BAD_PARAMETER_ERROR:
     45       return AudioProcessing::kBadParameterError;
     46     case AEC_BAD_PARAMETER_WARNING:
     47       return AudioProcessing::kBadStreamParameterWarning;
     48     default:
     49       // AEC_UNSPECIFIED_ERROR
     50       // AEC_UNINITIALIZED_ERROR
     51       // AEC_NULL_POINTER_ERROR
     52       return AudioProcessing::kUnspecifiedError;
     53   }
     54 }
     55 
     56 // Maximum length that a frame of samples can have.
     57 static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160;
     58 // Maximum number of frames to buffer in the render queue.
     59 // TODO(peah): Decrease this once we properly handle hugely unbalanced
     60 // reverse and forward call numbers.
     61 static const size_t kMaxNumFramesToBuffer = 100;
     62 }  // namespace
     63 
     64 EchoCancellationImpl::EchoCancellationImpl(const AudioProcessing* apm,
     65                                            rtc::CriticalSection* crit_render,
     66                                            rtc::CriticalSection* crit_capture)
     67     : ProcessingComponent(),
     68       apm_(apm),
     69       crit_render_(crit_render),
     70       crit_capture_(crit_capture),
     71       drift_compensation_enabled_(false),
     72       metrics_enabled_(false),
     73       suppression_level_(kModerateSuppression),
     74       stream_drift_samples_(0),
     75       was_stream_drift_set_(false),
     76       stream_has_echo_(false),
     77       delay_logging_enabled_(false),
     78       extended_filter_enabled_(false),
     79       delay_agnostic_enabled_(false),
     80       render_queue_element_max_size_(0) {
     81   RTC_DCHECK(apm);
     82   RTC_DCHECK(crit_render);
     83   RTC_DCHECK(crit_capture);
     84 }
     85 
     86 EchoCancellationImpl::~EchoCancellationImpl() {}
     87 
     88 int EchoCancellationImpl::ProcessRenderAudio(const AudioBuffer* audio) {
     89   rtc::CritScope cs_render(crit_render_);
     90   if (!is_component_enabled()) {
     91     return AudioProcessing::kNoError;
     92   }
     93 
     94   assert(audio->num_frames_per_band() <= 160);
     95   assert(audio->num_channels() == apm_->num_reverse_channels());
     96 
     97   int err = AudioProcessing::kNoError;
     98 
     99   // The ordering convention must be followed to pass to the correct AEC.
    100   size_t handle_index = 0;
    101   render_queue_buffer_.clear();
    102   for (size_t i = 0; i < apm_->num_output_channels(); i++) {
    103     for (size_t j = 0; j < audio->num_channels(); j++) {
    104       Handle* my_handle = static_cast<Handle*>(handle(handle_index));
    105       // Retrieve any error code produced by the buffering of the farend
    106       // signal
    107       err = WebRtcAec_GetBufferFarendError(
    108           my_handle, audio->split_bands_const_f(j)[kBand0To8kHz],
    109           audio->num_frames_per_band());
    110 
    111       if (err != AudioProcessing::kNoError) {
    112         return MapError(err);  // TODO(ajm): warning possible?
    113       }
    114 
    115       // Buffer the samples in the render queue.
    116       render_queue_buffer_.insert(render_queue_buffer_.end(),
    117                                   audio->split_bands_const_f(j)[kBand0To8kHz],
    118                                   (audio->split_bands_const_f(j)[kBand0To8kHz] +
    119                                    audio->num_frames_per_band()));
    120     }
    121   }
    122 
    123   // Insert the samples into the queue.
    124   if (!render_signal_queue_->Insert(&render_queue_buffer_)) {
    125     // The data queue is full and needs to be emptied.
    126     ReadQueuedRenderData();
    127 
    128     // Retry the insert (should always work).
    129     RTC_DCHECK_EQ(render_signal_queue_->Insert(&render_queue_buffer_), true);
    130   }
    131 
    132   return AudioProcessing::kNoError;
    133 }
    134 
    135 // Read chunks of data that were received and queued on the render side from
    136 // a queue. All the data chunks are buffered into the farend signal of the AEC.
    137 void EchoCancellationImpl::ReadQueuedRenderData() {
    138   rtc::CritScope cs_capture(crit_capture_);
    139   if (!is_component_enabled()) {
    140     return;
    141   }
    142 
    143   while (render_signal_queue_->Remove(&capture_queue_buffer_)) {
    144     size_t handle_index = 0;
    145     size_t buffer_index = 0;
    146     const size_t num_frames_per_band =
    147         capture_queue_buffer_.size() /
    148         (apm_->num_output_channels() * apm_->num_reverse_channels());
    149     for (size_t i = 0; i < apm_->num_output_channels(); i++) {
    150       for (size_t j = 0; j < apm_->num_reverse_channels(); j++) {
    151         Handle* my_handle = static_cast<Handle*>(handle(handle_index));
    152         WebRtcAec_BufferFarend(my_handle, &capture_queue_buffer_[buffer_index],
    153                                num_frames_per_band);
    154 
    155         buffer_index += num_frames_per_band;
    156         handle_index++;
    157       }
    158     }
    159   }
    160 }
    161 
    162 int EchoCancellationImpl::ProcessCaptureAudio(AudioBuffer* audio) {
    163   rtc::CritScope cs_capture(crit_capture_);
    164   if (!is_component_enabled()) {
    165     return AudioProcessing::kNoError;
    166   }
    167 
    168   if (!apm_->was_stream_delay_set()) {
    169     return AudioProcessing::kStreamParameterNotSetError;
    170   }
    171 
    172   if (drift_compensation_enabled_ && !was_stream_drift_set_) {
    173     return AudioProcessing::kStreamParameterNotSetError;
    174   }
    175 
    176   assert(audio->num_frames_per_band() <= 160);
    177   assert(audio->num_channels() == apm_->num_proc_channels());
    178 
    179   int err = AudioProcessing::kNoError;
    180 
    181   // The ordering convention must be followed to pass to the correct AEC.
    182   size_t handle_index = 0;
    183   stream_has_echo_ = false;
    184   for (size_t i = 0; i < audio->num_channels(); i++) {
    185     for (size_t j = 0; j < apm_->num_reverse_channels(); j++) {
    186       Handle* my_handle = handle(handle_index);
    187       err = WebRtcAec_Process(my_handle, audio->split_bands_const_f(i),
    188                               audio->num_bands(), audio->split_bands_f(i),
    189                               audio->num_frames_per_band(),
    190                               apm_->stream_delay_ms(), stream_drift_samples_);
    191 
    192       if (err != AudioProcessing::kNoError) {
    193         err = MapError(err);
    194         // TODO(ajm): Figure out how to return warnings properly.
    195         if (err != AudioProcessing::kBadStreamParameterWarning) {
    196           return err;
    197         }
    198       }
    199 
    200       int status = 0;
    201       err = WebRtcAec_get_echo_status(my_handle, &status);
    202       if (err != AudioProcessing::kNoError) {
    203         return MapError(err);
    204       }
    205 
    206       if (status == 1) {
    207         stream_has_echo_ = true;
    208       }
    209 
    210       handle_index++;
    211     }
    212   }
    213 
    214   was_stream_drift_set_ = false;
    215   return AudioProcessing::kNoError;
    216 }
    217 
    218 int EchoCancellationImpl::Enable(bool enable) {
    219   // Run in a single-threaded manner.
    220   rtc::CritScope cs_render(crit_render_);
    221   rtc::CritScope cs_capture(crit_capture_);
    222   // Ensure AEC and AECM are not both enabled.
    223   // The is_enabled call is safe from a deadlock perspective
    224   // as both locks are already held in the correct order.
    225   if (enable && apm_->echo_control_mobile()->is_enabled()) {
    226     return AudioProcessing::kBadParameterError;
    227   }
    228 
    229   return EnableComponent(enable);
    230 }
    231 
    232 bool EchoCancellationImpl::is_enabled() const {
    233   rtc::CritScope cs(crit_capture_);
    234   return is_component_enabled();
    235 }
    236 
    237 int EchoCancellationImpl::set_suppression_level(SuppressionLevel level) {
    238   {
    239     if (MapSetting(level) == -1) {
    240       return AudioProcessing::kBadParameterError;
    241     }
    242     rtc::CritScope cs(crit_capture_);
    243     suppression_level_ = level;
    244   }
    245   return Configure();
    246 }
    247 
    248 EchoCancellation::SuppressionLevel EchoCancellationImpl::suppression_level()
    249     const {
    250   rtc::CritScope cs(crit_capture_);
    251   return suppression_level_;
    252 }
    253 
    254 int EchoCancellationImpl::enable_drift_compensation(bool enable) {
    255   {
    256     rtc::CritScope cs(crit_capture_);
    257     drift_compensation_enabled_ = enable;
    258   }
    259   return Configure();
    260 }
    261 
    262 bool EchoCancellationImpl::is_drift_compensation_enabled() const {
    263   rtc::CritScope cs(crit_capture_);
    264   return drift_compensation_enabled_;
    265 }
    266 
    267 void EchoCancellationImpl::set_stream_drift_samples(int drift) {
    268   rtc::CritScope cs(crit_capture_);
    269   was_stream_drift_set_ = true;
    270   stream_drift_samples_ = drift;
    271 }
    272 
    273 int EchoCancellationImpl::stream_drift_samples() const {
    274   rtc::CritScope cs(crit_capture_);
    275   return stream_drift_samples_;
    276 }
    277 
    278 int EchoCancellationImpl::enable_metrics(bool enable) {
    279   {
    280     rtc::CritScope cs(crit_capture_);
    281     metrics_enabled_ = enable;
    282   }
    283   return Configure();
    284 }
    285 
    286 bool EchoCancellationImpl::are_metrics_enabled() const {
    287   rtc::CritScope cs(crit_capture_);
    288   return metrics_enabled_;
    289 }
    290 
    291 // TODO(ajm): we currently just use the metrics from the first AEC. Think more
    292 //            aboue the best way to extend this to multi-channel.
    293 int EchoCancellationImpl::GetMetrics(Metrics* metrics) {
    294   rtc::CritScope cs(crit_capture_);
    295   if (metrics == NULL) {
    296     return AudioProcessing::kNullPointerError;
    297   }
    298 
    299   if (!is_component_enabled() || !metrics_enabled_) {
    300     return AudioProcessing::kNotEnabledError;
    301   }
    302 
    303   AecMetrics my_metrics;
    304   memset(&my_metrics, 0, sizeof(my_metrics));
    305   memset(metrics, 0, sizeof(Metrics));
    306 
    307   Handle* my_handle = static_cast<Handle*>(handle(0));
    308   int err = WebRtcAec_GetMetrics(my_handle, &my_metrics);
    309   if (err != AudioProcessing::kNoError) {
    310     return MapError(err);
    311   }
    312 
    313   metrics->residual_echo_return_loss.instant = my_metrics.rerl.instant;
    314   metrics->residual_echo_return_loss.average = my_metrics.rerl.average;
    315   metrics->residual_echo_return_loss.maximum = my_metrics.rerl.max;
    316   metrics->residual_echo_return_loss.minimum = my_metrics.rerl.min;
    317 
    318   metrics->echo_return_loss.instant = my_metrics.erl.instant;
    319   metrics->echo_return_loss.average = my_metrics.erl.average;
    320   metrics->echo_return_loss.maximum = my_metrics.erl.max;
    321   metrics->echo_return_loss.minimum = my_metrics.erl.min;
    322 
    323   metrics->echo_return_loss_enhancement.instant = my_metrics.erle.instant;
    324   metrics->echo_return_loss_enhancement.average = my_metrics.erle.average;
    325   metrics->echo_return_loss_enhancement.maximum = my_metrics.erle.max;
    326   metrics->echo_return_loss_enhancement.minimum = my_metrics.erle.min;
    327 
    328   metrics->a_nlp.instant = my_metrics.aNlp.instant;
    329   metrics->a_nlp.average = my_metrics.aNlp.average;
    330   metrics->a_nlp.maximum = my_metrics.aNlp.max;
    331   metrics->a_nlp.minimum = my_metrics.aNlp.min;
    332 
    333   return AudioProcessing::kNoError;
    334 }
    335 
    336 bool EchoCancellationImpl::stream_has_echo() const {
    337   rtc::CritScope cs(crit_capture_);
    338   return stream_has_echo_;
    339 }
    340 
    341 int EchoCancellationImpl::enable_delay_logging(bool enable) {
    342   {
    343     rtc::CritScope cs(crit_capture_);
    344     delay_logging_enabled_ = enable;
    345   }
    346   return Configure();
    347 }
    348 
    349 bool EchoCancellationImpl::is_delay_logging_enabled() const {
    350   rtc::CritScope cs(crit_capture_);
    351   return delay_logging_enabled_;
    352 }
    353 
    354 bool EchoCancellationImpl::is_delay_agnostic_enabled() const {
    355   rtc::CritScope cs(crit_capture_);
    356   return delay_agnostic_enabled_;
    357 }
    358 
    359 bool EchoCancellationImpl::is_extended_filter_enabled() const {
    360   rtc::CritScope cs(crit_capture_);
    361   return extended_filter_enabled_;
    362 }
    363 
    364 // TODO(bjornv): How should we handle the multi-channel case?
    365 int EchoCancellationImpl::GetDelayMetrics(int* median, int* std) {
    366   rtc::CritScope cs(crit_capture_);
    367   float fraction_poor_delays = 0;
    368   return GetDelayMetrics(median, std, &fraction_poor_delays);
    369 }
    370 
    371 int EchoCancellationImpl::GetDelayMetrics(int* median, int* std,
    372                                           float* fraction_poor_delays) {
    373   rtc::CritScope cs(crit_capture_);
    374   if (median == NULL) {
    375     return AudioProcessing::kNullPointerError;
    376   }
    377   if (std == NULL) {
    378     return AudioProcessing::kNullPointerError;
    379   }
    380 
    381   if (!is_component_enabled() || !delay_logging_enabled_) {
    382     return AudioProcessing::kNotEnabledError;
    383   }
    384 
    385   Handle* my_handle = static_cast<Handle*>(handle(0));
    386   const int err =
    387       WebRtcAec_GetDelayMetrics(my_handle, median, std, fraction_poor_delays);
    388   if (err != AudioProcessing::kNoError) {
    389     return MapError(err);
    390   }
    391 
    392   return AudioProcessing::kNoError;
    393 }
    394 
    395 struct AecCore* EchoCancellationImpl::aec_core() const {
    396   rtc::CritScope cs(crit_capture_);
    397   if (!is_component_enabled()) {
    398     return NULL;
    399   }
    400   Handle* my_handle = static_cast<Handle*>(handle(0));
    401   return WebRtcAec_aec_core(my_handle);
    402 }
    403 
    404 int EchoCancellationImpl::Initialize() {
    405   int err = ProcessingComponent::Initialize();
    406   {
    407     rtc::CritScope cs(crit_capture_);
    408     if (err != AudioProcessing::kNoError || !is_component_enabled()) {
    409       return err;
    410     }
    411   }
    412 
    413   AllocateRenderQueue();
    414 
    415   return AudioProcessing::kNoError;
    416 }
    417 
    418 void EchoCancellationImpl::AllocateRenderQueue() {
    419   const size_t new_render_queue_element_max_size = std::max<size_t>(
    420       static_cast<size_t>(1),
    421       kMaxAllowedValuesOfSamplesPerFrame * num_handles_required());
    422 
    423   rtc::CritScope cs_render(crit_render_);
    424   rtc::CritScope cs_capture(crit_capture_);
    425 
    426   // Reallocate the queue if the queue item size is too small to fit the
    427   // data to put in the queue.
    428   if (render_queue_element_max_size_ < new_render_queue_element_max_size) {
    429     render_queue_element_max_size_ = new_render_queue_element_max_size;
    430 
    431     std::vector<float> template_queue_element(render_queue_element_max_size_);
    432 
    433     render_signal_queue_.reset(
    434         new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
    435             kMaxNumFramesToBuffer, template_queue_element,
    436             RenderQueueItemVerifier<float>(render_queue_element_max_size_)));
    437 
    438     render_queue_buffer_.resize(render_queue_element_max_size_);
    439     capture_queue_buffer_.resize(render_queue_element_max_size_);
    440   } else {
    441     render_signal_queue_->Clear();
    442   }
    443 }
    444 
    445 void EchoCancellationImpl::SetExtraOptions(const Config& config) {
    446   {
    447     rtc::CritScope cs(crit_capture_);
    448     extended_filter_enabled_ = config.Get<ExtendedFilter>().enabled;
    449     delay_agnostic_enabled_ = config.Get<DelayAgnostic>().enabled;
    450   }
    451   Configure();
    452 }
    453 
    454 void* EchoCancellationImpl::CreateHandle() const {
    455   return WebRtcAec_Create();
    456 }
    457 
    458 void EchoCancellationImpl::DestroyHandle(void* handle) const {
    459   assert(handle != NULL);
    460   WebRtcAec_Free(static_cast<Handle*>(handle));
    461 }
    462 
    463 int EchoCancellationImpl::InitializeHandle(void* handle) const {
    464   // Not locked as it only relies on APM public API which is threadsafe.
    465 
    466   assert(handle != NULL);
    467   // TODO(ajm): Drift compensation is disabled in practice. If restored, it
    468   // should be managed internally and not depend on the hardware sample rate.
    469   // For now, just hardcode a 48 kHz value.
    470   return WebRtcAec_Init(static_cast<Handle*>(handle),
    471                         apm_->proc_sample_rate_hz(), 48000);
    472 }
    473 
    474 int EchoCancellationImpl::ConfigureHandle(void* handle) const {
    475   rtc::CritScope cs_render(crit_render_);
    476   rtc::CritScope cs_capture(crit_capture_);
    477   assert(handle != NULL);
    478   AecConfig config;
    479   config.metricsMode = metrics_enabled_;
    480   config.nlpMode = MapSetting(suppression_level_);
    481   config.skewMode = drift_compensation_enabled_;
    482   config.delay_logging = delay_logging_enabled_;
    483   WebRtcAec_enable_extended_filter(
    484       WebRtcAec_aec_core(static_cast<Handle*>(handle)),
    485       extended_filter_enabled_ ? 1 : 0);
    486   WebRtcAec_enable_delay_agnostic(
    487       WebRtcAec_aec_core(static_cast<Handle*>(handle)),
    488       delay_agnostic_enabled_ ? 1 : 0);
    489   return WebRtcAec_set_config(static_cast<Handle*>(handle), config);
    490 }
    491 
    492 size_t EchoCancellationImpl::num_handles_required() const {
    493   // Not locked as it only relies on APM public API which is threadsafe.
    494   return apm_->num_output_channels() * apm_->num_reverse_channels();
    495 }
    496 
    497 int EchoCancellationImpl::GetHandleError(void* handle) const {
    498   // Not locked as it does not rely on anything in the state.
    499   assert(handle != NULL);
    500   return AudioProcessing::kUnspecifiedError;
    501 }
    502 }  // namespace webrtc
    503