1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "content/renderer/media/media_stream_audio_processor_options.h" 6 7 #include "base/files/file_path.h" 8 #include "base/files/file_util.h" 9 #include "base/logging.h" 10 #include "base/metrics/field_trial.h" 11 #include "base/path_service.h" 12 #include "base/strings/string_number_conversions.h" 13 #include "base/strings/utf_string_conversions.h" 14 #include "content/common/media/media_stream_options.h" 15 #include "content/renderer/media/media_stream_constraints_util.h" 16 #include "content/renderer/media/media_stream_source.h" 17 #include "content/renderer/media/rtc_media_constraints.h" 18 #include "media/audio/audio_parameters.h" 19 #include "third_party/webrtc/modules/audio_processing/include/audio_processing.h" 20 #include "third_party/webrtc/modules/audio_processing/typing_detection.h" 21 22 namespace content { 23 24 const char MediaAudioConstraints::kEchoCancellation[] = "echoCancellation"; 25 const char MediaAudioConstraints::kGoogEchoCancellation[] = 26 "googEchoCancellation"; 27 const char MediaAudioConstraints::kGoogExperimentalEchoCancellation[] = 28 "googEchoCancellation2"; 29 const char MediaAudioConstraints::kGoogAutoGainControl[] = 30 "googAutoGainControl"; 31 const char MediaAudioConstraints::kGoogExperimentalAutoGainControl[] = 32 "googAutoGainControl2"; 33 const char MediaAudioConstraints::kGoogNoiseSuppression[] = 34 "googNoiseSuppression"; 35 const char MediaAudioConstraints::kGoogExperimentalNoiseSuppression[] = 36 "googNoiseSuppression2"; 37 const char MediaAudioConstraints::kGoogHighpassFilter[] = "googHighpassFilter"; 38 const char MediaAudioConstraints::kGoogTypingNoiseDetection[] = 39 "googTypingNoiseDetection"; 40 const char MediaAudioConstraints::kGoogAudioMirroring[] = "googAudioMirroring"; 41 42 namespace { 43 44 // Constant constraint keys which enables default audio constraints on 45 // mediastreams with audio. 46 struct { 47 const char* key; 48 bool value; 49 } const kDefaultAudioConstraints[] = { 50 { MediaAudioConstraints::kEchoCancellation, true }, 51 { MediaAudioConstraints::kGoogEchoCancellation, true }, 52 #if defined(OS_ANDROID) || defined(OS_IOS) 53 { MediaAudioConstraints::kGoogExperimentalEchoCancellation, false }, 54 #else 55 // Enable the extended filter mode AEC on all non-mobile platforms. 56 { MediaAudioConstraints::kGoogExperimentalEchoCancellation, true }, 57 #endif 58 { MediaAudioConstraints::kGoogAutoGainControl, true }, 59 { MediaAudioConstraints::kGoogExperimentalAutoGainControl, true }, 60 { MediaAudioConstraints::kGoogNoiseSuppression, true }, 61 { MediaAudioConstraints::kGoogHighpassFilter, true }, 62 { MediaAudioConstraints::kGoogTypingNoiseDetection, true }, 63 { MediaAudioConstraints::kGoogExperimentalNoiseSuppression, false }, 64 #if defined(OS_WIN) 65 { kMediaStreamAudioDucking, true }, 66 #else 67 { kMediaStreamAudioDucking, false }, 68 #endif 69 }; 70 71 bool IsAudioProcessingConstraint(const std::string& key) { 72 // |kMediaStreamAudioDucking| does not require audio processing. 73 return key != kMediaStreamAudioDucking; 74 } 75 76 } // namespace 77 78 // TODO(xians): Remove this method after the APM in WebRtc is deprecated. 79 void MediaAudioConstraints::ApplyFixedAudioConstraints( 80 RTCMediaConstraints* constraints) { 81 for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kDefaultAudioConstraints); ++i) { 82 bool already_set_value; 83 if (!webrtc::FindConstraint(constraints, kDefaultAudioConstraints[i].key, 84 &already_set_value, NULL)) { 85 const std::string value = kDefaultAudioConstraints[i].value ? 86 webrtc::MediaConstraintsInterface::kValueTrue : 87 webrtc::MediaConstraintsInterface::kValueFalse; 88 constraints->AddOptional(kDefaultAudioConstraints[i].key, value, false); 89 } else { 90 DVLOG(1) << "Constraint " << kDefaultAudioConstraints[i].key 91 << " already set to " << already_set_value; 92 } 93 } 94 } 95 96 MediaAudioConstraints::MediaAudioConstraints( 97 const blink::WebMediaConstraints& constraints, int effects) 98 : constraints_(constraints), 99 effects_(effects), 100 default_audio_processing_constraint_value_(true) { 101 // The default audio processing constraints are turned off when 102 // - gUM has a specific kMediaStreamSource, which is used by tab capture 103 // and screen capture. 104 // - |kEchoCancellation| is explicitly set to false. 105 std::string value_str; 106 bool value_bool = false; 107 if ((GetConstraintValueAsString(constraints, kMediaStreamSource, 108 &value_str)) || 109 (GetConstraintValueAsBoolean(constraints_, kEchoCancellation, 110 &value_bool) && !value_bool)) { 111 default_audio_processing_constraint_value_ = false; 112 } 113 } 114 115 MediaAudioConstraints::~MediaAudioConstraints() {} 116 117 // TODO(xians): Remove this method after the APM in WebRtc is deprecated. 118 bool MediaAudioConstraints::NeedsAudioProcessing() { 119 if (GetEchoCancellationProperty()) 120 return true; 121 122 for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kDefaultAudioConstraints); ++i) { 123 // |kEchoCancellation| and |kGoogEchoCancellation| have been convered by 124 // GetEchoCancellationProperty(). 125 if (kDefaultAudioConstraints[i].key != kEchoCancellation && 126 kDefaultAudioConstraints[i].key != kGoogEchoCancellation && 127 IsAudioProcessingConstraint(kDefaultAudioConstraints[i].key) && 128 GetProperty(kDefaultAudioConstraints[i].key)) { 129 return true; 130 } 131 } 132 133 return false; 134 } 135 136 bool MediaAudioConstraints::GetProperty(const std::string& key) { 137 // Return the value if the constraint is specified in |constraints|, 138 // otherwise return the default value. 139 bool value = false; 140 if (!GetConstraintValueAsBoolean(constraints_, key, &value)) 141 value = GetDefaultValueForConstraint(constraints_, key); 142 143 return value; 144 } 145 146 bool MediaAudioConstraints::GetEchoCancellationProperty() { 147 // If platform echo canceller is enabled, disable the software AEC. 148 if (effects_ & media::AudioParameters::ECHO_CANCELLER) 149 return false; 150 151 // If |kEchoCancellation| is specified in the constraints, it will 152 // override the value of |kGoogEchoCancellation|. 153 bool value = false; 154 if (GetConstraintValueAsBoolean(constraints_, kEchoCancellation, &value)) 155 return value; 156 157 return GetProperty(kGoogEchoCancellation); 158 } 159 160 bool MediaAudioConstraints::IsValid() { 161 blink::WebVector<blink::WebMediaConstraint> mandatory; 162 constraints_.getMandatoryConstraints(mandatory); 163 for (size_t i = 0; i < mandatory.size(); ++i) { 164 const std::string key = mandatory[i].m_name.utf8(); 165 if (key == kMediaStreamSource || key == kMediaStreamSourceId || 166 key == MediaStreamSource::kSourceId) { 167 // Ignore Chrome specific Tab capture and |kSourceId| constraints. 168 continue; 169 } 170 171 bool valid = false; 172 for (size_t j = 0; j < ARRAYSIZE_UNSAFE(kDefaultAudioConstraints); ++j) { 173 if (key == kDefaultAudioConstraints[j].key) { 174 bool value = false; 175 valid = GetMandatoryConstraintValueAsBoolean(constraints_, key, &value); 176 break; 177 } 178 } 179 180 if (!valid) { 181 DLOG(ERROR) << "Invalid MediaStream constraint. Name: " << key; 182 return false; 183 } 184 } 185 186 return true; 187 } 188 189 bool MediaAudioConstraints::GetDefaultValueForConstraint( 190 const blink::WebMediaConstraints& constraints, const std::string& key) { 191 // |kMediaStreamAudioDucking| is not restricted by 192 // |default_audio_processing_constraint_value_| since it does not require 193 // audio processing. 194 if (!default_audio_processing_constraint_value_ && 195 IsAudioProcessingConstraint(key)) 196 return false; 197 198 for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kDefaultAudioConstraints); ++i) { 199 if (kDefaultAudioConstraints[i].key == key) 200 return kDefaultAudioConstraints[i].value; 201 } 202 203 return false; 204 } 205 206 void EnableEchoCancellation(AudioProcessing* audio_processing) { 207 #if defined(OS_ANDROID) || defined(OS_IOS) 208 const std::string group_name = 209 base::FieldTrialList::FindFullName("ReplaceAECMWithAEC"); 210 if (group_name.empty() || group_name != "Enabled") { 211 // Mobile devices are using AECM. 212 int err = audio_processing->echo_control_mobile()->set_routing_mode( 213 webrtc::EchoControlMobile::kSpeakerphone); 214 err |= audio_processing->echo_control_mobile()->Enable(true); 215 CHECK_EQ(err, 0); 216 return; 217 } 218 #endif 219 int err = audio_processing->echo_cancellation()->set_suppression_level( 220 webrtc::EchoCancellation::kHighSuppression); 221 222 // Enable the metrics for AEC. 223 err |= audio_processing->echo_cancellation()->enable_metrics(true); 224 err |= audio_processing->echo_cancellation()->enable_delay_logging(true); 225 err |= audio_processing->echo_cancellation()->Enable(true); 226 CHECK_EQ(err, 0); 227 } 228 229 void EnableNoiseSuppression(AudioProcessing* audio_processing) { 230 int err = audio_processing->noise_suppression()->set_level( 231 webrtc::NoiseSuppression::kHigh); 232 err |= audio_processing->noise_suppression()->Enable(true); 233 CHECK_EQ(err, 0); 234 } 235 236 void EnableHighPassFilter(AudioProcessing* audio_processing) { 237 CHECK_EQ(audio_processing->high_pass_filter()->Enable(true), 0); 238 } 239 240 void EnableTypingDetection(AudioProcessing* audio_processing, 241 webrtc::TypingDetection* typing_detector) { 242 int err = audio_processing->voice_detection()->Enable(true); 243 err |= audio_processing->voice_detection()->set_likelihood( 244 webrtc::VoiceDetection::kVeryLowLikelihood); 245 CHECK_EQ(err, 0); 246 247 // Configure the update period to 1s (100 * 10ms) in the typing detector. 248 typing_detector->SetParameters(0, 0, 0, 0, 0, 100); 249 } 250 251 void StartEchoCancellationDump(AudioProcessing* audio_processing, 252 base::File aec_dump_file) { 253 DCHECK(aec_dump_file.IsValid()); 254 255 FILE* stream = base::FileToFILE(aec_dump_file.Pass(), "w"); 256 if (!stream) { 257 LOG(ERROR) << "Failed to open AEC dump file"; 258 return; 259 } 260 261 if (audio_processing->StartDebugRecording(stream)) 262 DLOG(ERROR) << "Fail to start AEC debug recording"; 263 } 264 265 void StopEchoCancellationDump(AudioProcessing* audio_processing) { 266 if (audio_processing->StopDebugRecording()) 267 DLOG(ERROR) << "Fail to stop AEC debug recording"; 268 } 269 270 void EnableAutomaticGainControl(AudioProcessing* audio_processing) { 271 #if defined(OS_ANDROID) || defined(OS_IOS) 272 const webrtc::GainControl::Mode mode = webrtc::GainControl::kFixedDigital; 273 #else 274 const webrtc::GainControl::Mode mode = webrtc::GainControl::kAdaptiveAnalog; 275 #endif 276 int err = audio_processing->gain_control()->set_mode(mode); 277 err |= audio_processing->gain_control()->Enable(true); 278 CHECK_EQ(err, 0); 279 } 280 281 void GetAecStats(AudioProcessing* audio_processing, 282 webrtc::AudioProcessorInterface::AudioProcessorStats* stats) { 283 // These values can take on valid negative values, so use the lowest possible 284 // level as default rather than -1. 285 stats->echo_return_loss = -100; 286 stats->echo_return_loss_enhancement = -100; 287 288 // These values can also be negative, but in practice -1 is only used to 289 // signal insufficient data, since the resolution is limited to multiples 290 // of 4ms. 291 stats->echo_delay_median_ms = -1; 292 stats->echo_delay_std_ms = -1; 293 294 // TODO(ajm): Re-enable this metric once we have a reliable implementation. 295 stats->aec_quality_min = -1.0f; 296 297 if (!audio_processing->echo_cancellation()->are_metrics_enabled() || 298 !audio_processing->echo_cancellation()->is_delay_logging_enabled() || 299 !audio_processing->echo_cancellation()->is_enabled()) { 300 return; 301 } 302 303 // TODO(ajm): we may want to use VoECallReport::GetEchoMetricsSummary 304 // here, but it appears to be unsuitable currently. Revisit after this is 305 // investigated: http://b/issue?id=5666755 306 webrtc::EchoCancellation::Metrics echo_metrics; 307 if (!audio_processing->echo_cancellation()->GetMetrics(&echo_metrics)) { 308 stats->echo_return_loss = echo_metrics.echo_return_loss.instant; 309 stats->echo_return_loss_enhancement = 310 echo_metrics.echo_return_loss_enhancement.instant; 311 } 312 313 int median = 0, std = 0; 314 if (!audio_processing->echo_cancellation()->GetDelayMetrics(&median, &std)) { 315 stats->echo_delay_median_ms = median; 316 stats->echo_delay_std_ms = std; 317 } 318 } 319 320 } // namespace content 321