Home | History | Annotate | Download | only in media 
      1 // Copyright 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "content/renderer/media/media_stream_audio_processor_options.h"
      6 
      7 #include "base/file_util.h"
      8 #include "base/files/file_path.h"
      9 #include "base/logging.h"
     10 #include "base/metrics/field_trial.h"
     11 #include "base/path_service.h"
     12 #include "base/strings/string_number_conversions.h"
     13 #include "base/strings/utf_string_conversions.h"
     14 #include "content/common/media/media_stream_options.h"
     15 #include "content/renderer/media/media_stream_constraints_util.h"
     16 #include "content/renderer/media/media_stream_source.h"
     17 #include "content/renderer/media/rtc_media_constraints.h"
     18 #include "media/audio/audio_parameters.h"
     19 #include "third_party/webrtc/modules/audio_processing/include/audio_processing.h"
     20 #include "third_party/webrtc/modules/audio_processing/typing_detection.h"
     21 
     22 namespace content {
     23 
     24 const char MediaAudioConstraints::kEchoCancellation[] = "echoCancellation";
     25 const char MediaAudioConstraints::kGoogEchoCancellation[] =
     26     "googEchoCancellation";
     27 const char MediaAudioConstraints::kGoogExperimentalEchoCancellation[] =
     28     "googEchoCancellation2";
     29 const char MediaAudioConstraints::kGoogAutoGainControl[] =
     30     "googAutoGainControl";
     31 const char MediaAudioConstraints::kGoogExperimentalAutoGainControl[] =
     32     "googAutoGainControl2";
     33 const char MediaAudioConstraints::kGoogNoiseSuppression[] =
     34     "googNoiseSuppression";
     35 const char MediaAudioConstraints::kGoogExperimentalNoiseSuppression[] =
     36     "googNoiseSuppression2";
     37 const char MediaAudioConstraints::kGoogHighpassFilter[] = "googHighpassFilter";
     38 const char MediaAudioConstraints::kGoogTypingNoiseDetection[] =
     39     "googTypingNoiseDetection";
     40 const char MediaAudioConstraints::kGoogAudioMirroring[] = "googAudioMirroring";
     41 
     42 namespace {
     43 
     44 // Constant constraint keys which enables default audio constraints on
     45 // mediastreams with audio.
     46 struct {
     47   const char* key;
     48   bool value;
     49 } const kDefaultAudioConstraints[] = {
     50   { MediaAudioConstraints::kEchoCancellation, true },
     51   { MediaAudioConstraints::kGoogEchoCancellation, true },
     52 #if defined(OS_CHROMEOS) || defined(OS_MACOSX)
     53   // Enable the extended filter mode AEC on platforms with known echo issues.
     54   { MediaAudioConstraints::kGoogExperimentalEchoCancellation, true },
     55 #else
     56   { MediaAudioConstraints::kGoogExperimentalEchoCancellation, false },
     57 #endif
     58   { MediaAudioConstraints::kGoogAutoGainControl, true },
     59   { MediaAudioConstraints::kGoogExperimentalAutoGainControl, true },
     60   { MediaAudioConstraints::kGoogNoiseSuppression, true },
     61   { MediaAudioConstraints::kGoogHighpassFilter, true },
     62   { MediaAudioConstraints::kGoogTypingNoiseDetection, true },
     63   { MediaAudioConstraints::kGoogExperimentalNoiseSuppression, false },
     64 #if defined(OS_WIN)
     65   { kMediaStreamAudioDucking, true },
     66 #else
     67   { kMediaStreamAudioDucking, false },
     68 #endif
     69 };
     70 
     71 bool IsAudioProcessingConstraint(const std::string& key) {
     72   // |kMediaStreamAudioDucking| does not require audio processing.
     73   return key != kMediaStreamAudioDucking;
     74 }
     75 
     76 } // namespace
     77 
     78 // TODO(xians): Remove this method after the APM in WebRtc is deprecated.
     79 void MediaAudioConstraints::ApplyFixedAudioConstraints(
     80     RTCMediaConstraints* constraints) {
     81   for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kDefaultAudioConstraints); ++i) {
     82     bool already_set_value;
     83     if (!webrtc::FindConstraint(constraints, kDefaultAudioConstraints[i].key,
     84                                 &already_set_value, NULL)) {
     85       const std::string value = kDefaultAudioConstraints[i].value ?
     86           webrtc::MediaConstraintsInterface::kValueTrue :
     87           webrtc::MediaConstraintsInterface::kValueFalse;
     88       constraints->AddOptional(kDefaultAudioConstraints[i].key, value, false);
     89     } else {
     90       DVLOG(1) << "Constraint " << kDefaultAudioConstraints[i].key
     91                << " already set to " << already_set_value;
     92     }
     93   }
     94 }
     95 
     96 MediaAudioConstraints::MediaAudioConstraints(
     97     const blink::WebMediaConstraints& constraints, int effects)
     98     : constraints_(constraints),
     99       effects_(effects),
    100       default_audio_processing_constraint_value_(true) {
    101   // The default audio processing constraints are turned off when
    102   // - gUM has a specific kMediaStreamSource, which is used by tab capture
    103   //   and screen capture.
    104   // - |kEchoCancellation| is explicitly set to false.
    105   std::string value_str;
    106   bool value_bool = false;
    107   if ((GetConstraintValueAsString(constraints, kMediaStreamSource,
    108                                   &value_str)) ||
    109       (GetConstraintValueAsBoolean(constraints_, kEchoCancellation,
    110                                    &value_bool) && !value_bool)) {
    111     default_audio_processing_constraint_value_ = false;
    112   }
    113 }
    114 
    115 MediaAudioConstraints::~MediaAudioConstraints() {}
    116 
    117 // TODO(xians): Remove this method after the APM in WebRtc is deprecated.
    118 bool MediaAudioConstraints::NeedsAudioProcessing() {
    119   if (GetEchoCancellationProperty())
    120     return true;
    121 
    122   for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kDefaultAudioConstraints); ++i) {
    123     // |kEchoCancellation| and |kGoogEchoCancellation| have been convered by
    124     // GetEchoCancellationProperty().
    125     if (kDefaultAudioConstraints[i].key != kEchoCancellation &&
    126         kDefaultAudioConstraints[i].key != kGoogEchoCancellation &&
    127         IsAudioProcessingConstraint(kDefaultAudioConstraints[i].key) &&
    128         GetProperty(kDefaultAudioConstraints[i].key)) {
    129       return true;
    130     }
    131   }
    132 
    133   return false;
    134 }
    135 
    136 bool MediaAudioConstraints::GetProperty(const std::string& key) {
    137   // Return the value if the constraint is specified in |constraints|,
    138   // otherwise return the default value.
    139   bool value = false;
    140   if (!GetConstraintValueAsBoolean(constraints_, key, &value))
    141     value = GetDefaultValueForConstraint(constraints_, key);
    142 
    143   return value;
    144 }
    145 
    146 bool MediaAudioConstraints::GetEchoCancellationProperty() {
    147   // If platform echo canceller is enabled, disable the software AEC.
    148   if (effects_ & media::AudioParameters::ECHO_CANCELLER)
    149     return false;
    150 
    151   // If |kEchoCancellation| is specified in the constraints, it will
    152   // override the value of |kGoogEchoCancellation|.
    153   bool value = false;
    154   if (GetConstraintValueAsBoolean(constraints_, kEchoCancellation, &value))
    155     return value;
    156 
    157   return GetProperty(kGoogEchoCancellation);
    158 }
    159 
    160 bool MediaAudioConstraints::IsValid() {
    161   blink::WebVector<blink::WebMediaConstraint> mandatory;
    162   constraints_.getMandatoryConstraints(mandatory);
    163   for (size_t i = 0; i < mandatory.size(); ++i) {
    164     const std::string key = mandatory[i].m_name.utf8();
    165     if (key == kMediaStreamSource || key == kMediaStreamSourceId ||
    166         key == MediaStreamSource::kSourceId) {
    167       // Ignore Chrome specific Tab capture and |kSourceId| constraints.
    168       continue;
    169     }
    170 
    171     bool valid = false;
    172     for (size_t j = 0; j < ARRAYSIZE_UNSAFE(kDefaultAudioConstraints); ++j) {
    173       if (key == kDefaultAudioConstraints[j].key) {
    174         bool value = false;
    175         valid = GetMandatoryConstraintValueAsBoolean(constraints_, key, &value);
    176         break;
    177       }
    178     }
    179 
    180     if (!valid) {
    181       DLOG(ERROR) << "Invalid MediaStream constraint. Name: " << key;
    182       return false;
    183     }
    184   }
    185 
    186   return true;
    187 }
    188 
    189 bool MediaAudioConstraints::GetDefaultValueForConstraint(
    190     const blink::WebMediaConstraints& constraints, const std::string& key) {
    191   // |kMediaStreamAudioDucking| is not restricted by
    192   // |default_audio_processing_constraint_value_| since it does not require
    193   // audio processing.
    194   if (!default_audio_processing_constraint_value_ &&
    195       IsAudioProcessingConstraint(key))
    196     return false;
    197 
    198   for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kDefaultAudioConstraints); ++i) {
    199     if (kDefaultAudioConstraints[i].key == key)
    200       return kDefaultAudioConstraints[i].value;
    201   }
    202 
    203   return false;
    204 }
    205 
    206 void EnableEchoCancellation(AudioProcessing* audio_processing) {
    207 #if defined(OS_ANDROID) || defined(OS_IOS)
    208   const std::string group_name =
    209       base::FieldTrialList::FindFullName("ReplaceAECMWithAEC");
    210   if (group_name.empty() || group_name != "Enabled") {
    211     // Mobile devices are using AECM.
    212     int err = audio_processing->echo_control_mobile()->set_routing_mode(
    213         webrtc::EchoControlMobile::kSpeakerphone);
    214     err |= audio_processing->echo_control_mobile()->Enable(true);
    215     CHECK_EQ(err, 0);
    216     return;
    217   }
    218 #endif
    219   int err = audio_processing->echo_cancellation()->set_suppression_level(
    220       webrtc::EchoCancellation::kHighSuppression);
    221 
    222   // Enable the metrics for AEC.
    223   err |= audio_processing->echo_cancellation()->enable_metrics(true);
    224   err |= audio_processing->echo_cancellation()->enable_delay_logging(true);
    225   err |= audio_processing->echo_cancellation()->Enable(true);
    226   CHECK_EQ(err, 0);
    227 }
    228 
    229 void EnableNoiseSuppression(AudioProcessing* audio_processing) {
    230   int err = audio_processing->noise_suppression()->set_level(
    231       webrtc::NoiseSuppression::kHigh);
    232   err |= audio_processing->noise_suppression()->Enable(true);
    233   CHECK_EQ(err, 0);
    234 }
    235 
    236 void EnableExperimentalNoiseSuppression(AudioProcessing* audio_processing) {
    237   CHECK_EQ(audio_processing->EnableExperimentalNs(true), 0);
    238 }
    239 
    240 void EnableHighPassFilter(AudioProcessing* audio_processing) {
    241   CHECK_EQ(audio_processing->high_pass_filter()->Enable(true), 0);
    242 }
    243 
    244 void EnableTypingDetection(AudioProcessing* audio_processing,
    245                            webrtc::TypingDetection* typing_detector) {
    246   int err = audio_processing->voice_detection()->Enable(true);
    247   err |= audio_processing->voice_detection()->set_likelihood(
    248       webrtc::VoiceDetection::kVeryLowLikelihood);
    249   CHECK_EQ(err, 0);
    250 
    251   // Configure the update period to 1s (100 * 10ms) in the typing detector.
    252   typing_detector->SetParameters(0, 0, 0, 0, 0, 100);
    253 }
    254 
    255 void EnableExperimentalEchoCancellation(AudioProcessing* audio_processing) {
    256   webrtc::Config config;
    257   config.Set<webrtc::DelayCorrection>(new webrtc::DelayCorrection(true));
    258   audio_processing->SetExtraOptions(config);
    259 }
    260 
    261 void StartEchoCancellationDump(AudioProcessing* audio_processing,
    262                                base::File aec_dump_file) {
    263   DCHECK(aec_dump_file.IsValid());
    264 
    265   FILE* stream = base::FileToFILE(aec_dump_file.Pass(), "w");
    266   if (!stream) {
    267     LOG(ERROR) << "Failed to open AEC dump file";
    268     return;
    269   }
    270 
    271   if (audio_processing->StartDebugRecording(stream))
    272     DLOG(ERROR) << "Fail to start AEC debug recording";
    273 }
    274 
    275 void StopEchoCancellationDump(AudioProcessing* audio_processing) {
    276   if (audio_processing->StopDebugRecording())
    277     DLOG(ERROR) << "Fail to stop AEC debug recording";
    278 }
    279 
    280 void EnableAutomaticGainControl(AudioProcessing* audio_processing) {
    281 #if defined(OS_ANDROID) || defined(OS_IOS)
    282   const webrtc::GainControl::Mode mode = webrtc::GainControl::kFixedDigital;
    283 #else
    284   const webrtc::GainControl::Mode mode = webrtc::GainControl::kAdaptiveAnalog;
    285 #endif
    286   int err = audio_processing->gain_control()->set_mode(mode);
    287   err |= audio_processing->gain_control()->Enable(true);
    288   CHECK_EQ(err, 0);
    289 }
    290 
    291 void GetAecStats(AudioProcessing* audio_processing,
    292                  webrtc::AudioProcessorInterface::AudioProcessorStats* stats) {
    293   // These values can take on valid negative values, so use the lowest possible
    294   // level as default rather than -1.
    295   stats->echo_return_loss = -100;
    296   stats->echo_return_loss_enhancement = -100;
    297 
    298   // These values can also be negative, but in practice -1 is only used to
    299   // signal insufficient data, since the resolution is limited to multiples
    300   // of 4ms.
    301   stats->echo_delay_median_ms = -1;
    302   stats->echo_delay_std_ms = -1;
    303 
    304   // TODO(ajm): Re-enable this metric once we have a reliable implementation.
    305   stats->aec_quality_min = -1.0f;
    306 
    307   if (!audio_processing->echo_cancellation()->are_metrics_enabled() ||
    308       !audio_processing->echo_cancellation()->is_delay_logging_enabled() ||
    309       !audio_processing->echo_cancellation()->is_enabled()) {
    310     return;
    311   }
    312 
    313   // TODO(ajm): we may want to use VoECallReport::GetEchoMetricsSummary
    314   // here, but it appears to be unsuitable currently. Revisit after this is
    315   // investigated: http://b/issue?id=5666755
    316   webrtc::EchoCancellation::Metrics echo_metrics;
    317   if (!audio_processing->echo_cancellation()->GetMetrics(&echo_metrics)) {
    318     stats->echo_return_loss = echo_metrics.echo_return_loss.instant;
    319     stats->echo_return_loss_enhancement =
    320         echo_metrics.echo_return_loss_enhancement.instant;
    321   }
    322 
    323   int median = 0, std = 0;
    324   if (!audio_processing->echo_cancellation()->GetDelayMetrics(&median, &std)) {
    325     stats->echo_delay_median_ms = median;
    326     stats->echo_delay_std_ms = std;
    327   }
    328 }
    329 
    330 }  // namespace content
    331