1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ 12 #define WEBRTC_MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ 13 14 #include <stddef.h> // size_t 15 #include <stdio.h> // FILE 16 17 #include "webrtc/common.h" 18 #include "webrtc/typedefs.h" 19 20 struct AecCore; 21 22 namespace webrtc { 23 24 class AudioFrame; 25 class EchoCancellation; 26 class EchoControlMobile; 27 class GainControl; 28 class HighPassFilter; 29 class LevelEstimator; 30 class NoiseSuppression; 31 class VoiceDetection; 32 33 // Use to enable the delay correction feature. This now engages an extended 34 // filter mode in the AEC, along with robustness measures around the reported 35 // system delays. It comes with a significant increase in AEC complexity, but is 36 // much more robust to unreliable reported delays. 37 // 38 // Detailed changes to the algorithm: 39 // - The filter length is changed from 48 to 128 ms. This comes with tuning of 40 // several parameters: i) filter adaptation stepsize and error threshold; 41 // ii) non-linear processing smoothing and overdrive. 42 // - Option to ignore the reported delays on platforms which we deem 43 // sufficiently unreliable. See WEBRTC_UNTRUSTED_DELAY in echo_cancellation.c. 44 // - Faster startup times by removing the excessive "startup phase" processing 45 // of reported delays. 46 // - Much more conservative adjustments to the far-end read pointer. We smooth 47 // the delay difference more heavily, and back off from the difference more. 48 // Adjustments force a readaptation of the filter, so they should be avoided 49 // except when really necessary. 50 struct DelayCorrection { 51 DelayCorrection() : enabled(false) {} 52 explicit DelayCorrection(bool enabled) : enabled(enabled) {} 53 bool enabled; 54 }; 55 56 // Use to disable the reported system delays. By disabling the reported system 57 // delays the echo cancellation algorithm assumes the process and reverse 58 // streams to be aligned. This configuration only applies to EchoCancellation 59 // and not EchoControlMobile and is set with AudioProcessing::SetExtraOptions(). 60 // Note that by disabling reported system delays the EchoCancellation may 61 // regress in performance. 62 struct ReportedDelay { 63 ReportedDelay() : enabled(true) {} 64 explicit ReportedDelay(bool enabled) : enabled(enabled) {} 65 bool enabled; 66 }; 67 68 // Must be provided through AudioProcessing::Create(Confg&). It will have no 69 // impact if used with AudioProcessing::SetExtraOptions(). 70 struct ExperimentalAgc { 71 ExperimentalAgc() : enabled(true) {} 72 explicit ExperimentalAgc(bool enabled) : enabled(enabled) {} 73 bool enabled; 74 }; 75 76 static const int kAudioProcMaxNativeSampleRateHz = 32000; 77 78 // The Audio Processing Module (APM) provides a collection of voice processing 79 // components designed for real-time communications software. 80 // 81 // APM operates on two audio streams on a frame-by-frame basis. Frames of the 82 // primary stream, on which all processing is applied, are passed to 83 // |ProcessStream()|. Frames of the reverse direction stream, which are used for 84 // analysis by some components, are passed to |AnalyzeReverseStream()|. On the 85 // client-side, this will typically be the near-end (capture) and far-end 86 // (render) streams, respectively. APM should be placed in the signal chain as 87 // close to the audio hardware abstraction layer (HAL) as possible. 88 // 89 // On the server-side, the reverse stream will normally not be used, with 90 // processing occurring on each incoming stream. 91 // 92 // Component interfaces follow a similar pattern and are accessed through 93 // corresponding getters in APM. All components are disabled at create-time, 94 // with default settings that are recommended for most situations. New settings 95 // can be applied without enabling a component. Enabling a component triggers 96 // memory allocation and initialization to allow it to start processing the 97 // streams. 98 // 99 // Thread safety is provided with the following assumptions to reduce locking 100 // overhead: 101 // 1. The stream getters and setters are called from the same thread as 102 // ProcessStream(). More precisely, stream functions are never called 103 // concurrently with ProcessStream(). 104 // 2. Parameter getters are never called concurrently with the corresponding 105 // setter. 106 // 107 // APM accepts only linear PCM audio data in chunks of 10 ms. The int16 108 // interfaces use interleaved data, while the float interfaces use deinterleaved 109 // data. 110 // 111 // Usage example, omitting error checking: 112 // AudioProcessing* apm = AudioProcessing::Create(0); 113 // 114 // apm->high_pass_filter()->Enable(true); 115 // 116 // apm->echo_cancellation()->enable_drift_compensation(false); 117 // apm->echo_cancellation()->Enable(true); 118 // 119 // apm->noise_reduction()->set_level(kHighSuppression); 120 // apm->noise_reduction()->Enable(true); 121 // 122 // apm->gain_control()->set_analog_level_limits(0, 255); 123 // apm->gain_control()->set_mode(kAdaptiveAnalog); 124 // apm->gain_control()->Enable(true); 125 // 126 // apm->voice_detection()->Enable(true); 127 // 128 // // Start a voice call... 129 // 130 // // ... Render frame arrives bound for the audio HAL ... 131 // apm->AnalyzeReverseStream(render_frame); 132 // 133 // // ... Capture frame arrives from the audio HAL ... 134 // // Call required set_stream_ functions. 135 // apm->set_stream_delay_ms(delay_ms); 136 // apm->gain_control()->set_stream_analog_level(analog_level); 137 // 138 // apm->ProcessStream(capture_frame); 139 // 140 // // Call required stream_ functions. 141 // analog_level = apm->gain_control()->stream_analog_level(); 142 // has_voice = apm->stream_has_voice(); 143 // 144 // // Repeate render and capture processing for the duration of the call... 145 // // Start a new call... 146 // apm->Initialize(); 147 // 148 // // Close the application... 149 // delete apm; 150 // 151 class AudioProcessing { 152 public: 153 enum ChannelLayout { 154 kMono, 155 // Left, right. 156 kStereo, 157 // Mono, keyboard mic. 158 kMonoAndKeyboard, 159 // Left, right, keyboard mic. 160 kStereoAndKeyboard 161 }; 162 163 // Creates an APM instance. Use one instance for every primary audio stream 164 // requiring processing. On the client-side, this would typically be one 165 // instance for the near-end stream, and additional instances for each far-end 166 // stream which requires processing. On the server-side, this would typically 167 // be one instance for every incoming stream. 168 static AudioProcessing* Create(); 169 // Allows passing in an optional configuration at create-time. 170 static AudioProcessing* Create(const Config& config); 171 // TODO(ajm): Deprecated; remove all calls to it. 172 static AudioProcessing* Create(int id); 173 virtual ~AudioProcessing() {} 174 175 // Initializes internal states, while retaining all user settings. This 176 // should be called before beginning to process a new audio stream. However, 177 // it is not necessary to call before processing the first stream after 178 // creation. 179 // 180 // It is also not necessary to call if the audio parameters (sample 181 // rate and number of channels) have changed. Passing updated parameters 182 // directly to |ProcessStream()| and |AnalyzeReverseStream()| is permissible. 183 // If the parameters are known at init-time though, they may be provided. 184 virtual int Initialize() = 0; 185 186 // The int16 interfaces require: 187 // - only |NativeRate|s be used 188 // - that the input, output and reverse rates must match 189 // - that |output_layout| matches |input_layout| 190 // 191 // The float interfaces accept arbitrary rates and support differing input 192 // and output layouts, but the output may only remove channels, not add. 193 virtual int Initialize(int input_sample_rate_hz, 194 int output_sample_rate_hz, 195 int reverse_sample_rate_hz, 196 ChannelLayout input_layout, 197 ChannelLayout output_layout, 198 ChannelLayout reverse_layout) = 0; 199 200 // Pass down additional options which don't have explicit setters. This 201 // ensures the options are applied immediately. 202 virtual void SetExtraOptions(const Config& config) = 0; 203 204 virtual int EnableExperimentalNs(bool enable) = 0; 205 virtual bool experimental_ns_enabled() const = 0; 206 207 // DEPRECATED. 208 // TODO(ajm): Remove after Chromium has upgraded to using Initialize(). 209 virtual int set_sample_rate_hz(int rate) = 0; 210 // TODO(ajm): Remove after voice engine no longer requires it to resample 211 // the reverse stream to the forward rate. 212 virtual int input_sample_rate_hz() const = 0; 213 // TODO(ajm): Remove after Chromium no longer depends on it. 214 virtual int sample_rate_hz() const = 0; 215 216 // TODO(ajm): Only intended for internal use. Make private and friend the 217 // necessary classes? 218 virtual int proc_sample_rate_hz() const = 0; 219 virtual int proc_split_sample_rate_hz() const = 0; 220 virtual int num_input_channels() const = 0; 221 virtual int num_output_channels() const = 0; 222 virtual int num_reverse_channels() const = 0; 223 224 // Set to true when the output of AudioProcessing will be muted or in some 225 // other way not used. Ideally, the captured audio would still be processed, 226 // but some components may change behavior based on this information. 227 // Default false. 228 virtual void set_output_will_be_muted(bool muted) = 0; 229 virtual bool output_will_be_muted() const = 0; 230 231 // Processes a 10 ms |frame| of the primary audio stream. On the client-side, 232 // this is the near-end (or captured) audio. 233 // 234 // If needed for enabled functionality, any function with the set_stream_ tag 235 // must be called prior to processing the current frame. Any getter function 236 // with the stream_ tag which is needed should be called after processing. 237 // 238 // The |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_| 239 // members of |frame| must be valid. If changed from the previous call to this 240 // method, it will trigger an initialization. 241 virtual int ProcessStream(AudioFrame* frame) = 0; 242 243 // Accepts deinterleaved float audio with the range [-1, 1]. Each element 244 // of |src| points to a channel buffer, arranged according to 245 // |input_layout|. At output, the channels will be arranged according to 246 // |output_layout| at |output_sample_rate_hz| in |dest|. 247 // 248 // The output layout may only remove channels, not add. |src| and |dest| 249 // may use the same memory, if desired. 250 virtual int ProcessStream(const float* const* src, 251 int samples_per_channel, 252 int input_sample_rate_hz, 253 ChannelLayout input_layout, 254 int output_sample_rate_hz, 255 ChannelLayout output_layout, 256 float* const* dest) = 0; 257 258 // Analyzes a 10 ms |frame| of the reverse direction audio stream. The frame 259 // will not be modified. On the client-side, this is the far-end (or to be 260 // rendered) audio. 261 // 262 // It is only necessary to provide this if echo processing is enabled, as the 263 // reverse stream forms the echo reference signal. It is recommended, but not 264 // necessary, to provide if gain control is enabled. On the server-side this 265 // typically will not be used. If you're not sure what to pass in here, 266 // chances are you don't need to use it. 267 // 268 // The |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_| 269 // members of |frame| must be valid. |sample_rate_hz_| must correspond to 270 // |input_sample_rate_hz()| 271 // 272 // TODO(ajm): add const to input; requires an implementation fix. 273 virtual int AnalyzeReverseStream(AudioFrame* frame) = 0; 274 275 // Accepts deinterleaved float audio with the range [-1, 1]. Each element 276 // of |data| points to a channel buffer, arranged according to |layout|. 277 virtual int AnalyzeReverseStream(const float* const* data, 278 int samples_per_channel, 279 int sample_rate_hz, 280 ChannelLayout layout) = 0; 281 282 // This must be called if and only if echo processing is enabled. 283 // 284 // Sets the |delay| in ms between AnalyzeReverseStream() receiving a far-end 285 // frame and ProcessStream() receiving a near-end frame containing the 286 // corresponding echo. On the client-side this can be expressed as 287 // delay = (t_render - t_analyze) + (t_process - t_capture) 288 // where, 289 // - t_analyze is the time a frame is passed to AnalyzeReverseStream() and 290 // t_render is the time the first sample of the same frame is rendered by 291 // the audio hardware. 292 // - t_capture is the time the first sample of a frame is captured by the 293 // audio hardware and t_pull is the time the same frame is passed to 294 // ProcessStream(). 295 virtual int set_stream_delay_ms(int delay) = 0; 296 virtual int stream_delay_ms() const = 0; 297 virtual bool was_stream_delay_set() const = 0; 298 299 // Call to signal that a key press occurred (true) or did not occur (false) 300 // with this chunk of audio. 301 virtual void set_stream_key_pressed(bool key_pressed) = 0; 302 virtual bool stream_key_pressed() const = 0; 303 304 // Sets a delay |offset| in ms to add to the values passed in through 305 // set_stream_delay_ms(). May be positive or negative. 306 // 307 // Note that this could cause an otherwise valid value passed to 308 // set_stream_delay_ms() to return an error. 309 virtual void set_delay_offset_ms(int offset) = 0; 310 virtual int delay_offset_ms() const = 0; 311 312 // Starts recording debugging information to a file specified by |filename|, 313 // a NULL-terminated string. If there is an ongoing recording, the old file 314 // will be closed, and recording will continue in the newly specified file. 315 // An already existing file will be overwritten without warning. 316 static const size_t kMaxFilenameSize = 1024; 317 virtual int StartDebugRecording(const char filename[kMaxFilenameSize]) = 0; 318 319 // Same as above but uses an existing file handle. Takes ownership 320 // of |handle| and closes it at StopDebugRecording(). 321 virtual int StartDebugRecording(FILE* handle) = 0; 322 323 // Stops recording debugging information, and closes the file. Recording 324 // cannot be resumed in the same file (without overwriting it). 325 virtual int StopDebugRecording() = 0; 326 327 // These provide access to the component interfaces and should never return 328 // NULL. The pointers will be valid for the lifetime of the APM instance. 329 // The memory for these objects is entirely managed internally. 330 virtual EchoCancellation* echo_cancellation() const = 0; 331 virtual EchoControlMobile* echo_control_mobile() const = 0; 332 virtual GainControl* gain_control() const = 0; 333 virtual HighPassFilter* high_pass_filter() const = 0; 334 virtual LevelEstimator* level_estimator() const = 0; 335 virtual NoiseSuppression* noise_suppression() const = 0; 336 virtual VoiceDetection* voice_detection() const = 0; 337 338 struct Statistic { 339 int instant; // Instantaneous value. 340 int average; // Long-term average. 341 int maximum; // Long-term maximum. 342 int minimum; // Long-term minimum. 343 }; 344 345 enum Error { 346 // Fatal errors. 347 kNoError = 0, 348 kUnspecifiedError = -1, 349 kCreationFailedError = -2, 350 kUnsupportedComponentError = -3, 351 kUnsupportedFunctionError = -4, 352 kNullPointerError = -5, 353 kBadParameterError = -6, 354 kBadSampleRateError = -7, 355 kBadDataLengthError = -8, 356 kBadNumberChannelsError = -9, 357 kFileError = -10, 358 kStreamParameterNotSetError = -11, 359 kNotEnabledError = -12, 360 361 // Warnings are non-fatal. 362 // This results when a set_stream_ parameter is out of range. Processing 363 // will continue, but the parameter may have been truncated. 364 kBadStreamParameterWarning = -13 365 }; 366 367 enum NativeRate { 368 kSampleRate8kHz = 8000, 369 kSampleRate16kHz = 16000, 370 kSampleRate32kHz = 32000 371 }; 372 373 static const int kChunkSizeMs = 10; 374 }; 375 376 // The acoustic echo cancellation (AEC) component provides better performance 377 // than AECM but also requires more processing power and is dependent on delay 378 // stability and reporting accuracy. As such it is well-suited and recommended 379 // for PC and IP phone applications. 380 // 381 // Not recommended to be enabled on the server-side. 382 class EchoCancellation { 383 public: 384 // EchoCancellation and EchoControlMobile may not be enabled simultaneously. 385 // Enabling one will disable the other. 386 virtual int Enable(bool enable) = 0; 387 virtual bool is_enabled() const = 0; 388 389 // Differences in clock speed on the primary and reverse streams can impact 390 // the AEC performance. On the client-side, this could be seen when different 391 // render and capture devices are used, particularly with webcams. 392 // 393 // This enables a compensation mechanism, and requires that 394 // set_stream_drift_samples() be called. 395 virtual int enable_drift_compensation(bool enable) = 0; 396 virtual bool is_drift_compensation_enabled() const = 0; 397 398 // Sets the difference between the number of samples rendered and captured by 399 // the audio devices since the last call to |ProcessStream()|. Must be called 400 // if drift compensation is enabled, prior to |ProcessStream()|. 401 virtual void set_stream_drift_samples(int drift) = 0; 402 virtual int stream_drift_samples() const = 0; 403 404 enum SuppressionLevel { 405 kLowSuppression, 406 kModerateSuppression, 407 kHighSuppression 408 }; 409 410 // Sets the aggressiveness of the suppressor. A higher level trades off 411 // double-talk performance for increased echo suppression. 412 virtual int set_suppression_level(SuppressionLevel level) = 0; 413 virtual SuppressionLevel suppression_level() const = 0; 414 415 // Returns false if the current frame almost certainly contains no echo 416 // and true if it _might_ contain echo. 417 virtual bool stream_has_echo() const = 0; 418 419 // Enables the computation of various echo metrics. These are obtained 420 // through |GetMetrics()|. 421 virtual int enable_metrics(bool enable) = 0; 422 virtual bool are_metrics_enabled() const = 0; 423 424 // Each statistic is reported in dB. 425 // P_far: Far-end (render) signal power. 426 // P_echo: Near-end (capture) echo signal power. 427 // P_out: Signal power at the output of the AEC. 428 // P_a: Internal signal power at the point before the AEC's non-linear 429 // processor. 430 struct Metrics { 431 // RERL = ERL + ERLE 432 AudioProcessing::Statistic residual_echo_return_loss; 433 434 // ERL = 10log_10(P_far / P_echo) 435 AudioProcessing::Statistic echo_return_loss; 436 437 // ERLE = 10log_10(P_echo / P_out) 438 AudioProcessing::Statistic echo_return_loss_enhancement; 439 440 // (Pre non-linear processing suppression) A_NLP = 10log_10(P_echo / P_a) 441 AudioProcessing::Statistic a_nlp; 442 }; 443 444 // TODO(ajm): discuss the metrics update period. 445 virtual int GetMetrics(Metrics* metrics) = 0; 446 447 // Enables computation and logging of delay values. Statistics are obtained 448 // through |GetDelayMetrics()|. 449 virtual int enable_delay_logging(bool enable) = 0; 450 virtual bool is_delay_logging_enabled() const = 0; 451 452 // The delay metrics consists of the delay |median| and the delay standard 453 // deviation |std|. The values are averaged over the time period since the 454 // last call to |GetDelayMetrics()|. 455 virtual int GetDelayMetrics(int* median, int* std) = 0; 456 457 // Returns a pointer to the low level AEC component. In case of multiple 458 // channels, the pointer to the first one is returned. A NULL pointer is 459 // returned when the AEC component is disabled or has not been initialized 460 // successfully. 461 virtual struct AecCore* aec_core() const = 0; 462 463 protected: 464 virtual ~EchoCancellation() {} 465 }; 466 467 // The acoustic echo control for mobile (AECM) component is a low complexity 468 // robust option intended for use on mobile devices. 469 // 470 // Not recommended to be enabled on the server-side. 471 class EchoControlMobile { 472 public: 473 // EchoCancellation and EchoControlMobile may not be enabled simultaneously. 474 // Enabling one will disable the other. 475 virtual int Enable(bool enable) = 0; 476 virtual bool is_enabled() const = 0; 477 478 // Recommended settings for particular audio routes. In general, the louder 479 // the echo is expected to be, the higher this value should be set. The 480 // preferred setting may vary from device to device. 481 enum RoutingMode { 482 kQuietEarpieceOrHeadset, 483 kEarpiece, 484 kLoudEarpiece, 485 kSpeakerphone, 486 kLoudSpeakerphone 487 }; 488 489 // Sets echo control appropriate for the audio routing |mode| on the device. 490 // It can and should be updated during a call if the audio routing changes. 491 virtual int set_routing_mode(RoutingMode mode) = 0; 492 virtual RoutingMode routing_mode() const = 0; 493 494 // Comfort noise replaces suppressed background noise to maintain a 495 // consistent signal level. 496 virtual int enable_comfort_noise(bool enable) = 0; 497 virtual bool is_comfort_noise_enabled() const = 0; 498 499 // A typical use case is to initialize the component with an echo path from a 500 // previous call. The echo path is retrieved using |GetEchoPath()|, typically 501 // at the end of a call. The data can then be stored for later use as an 502 // initializer before the next call, using |SetEchoPath()|. 503 // 504 // Controlling the echo path this way requires the data |size_bytes| to match 505 // the internal echo path size. This size can be acquired using 506 // |echo_path_size_bytes()|. |SetEchoPath()| causes an entire reset, worth 507 // noting if it is to be called during an ongoing call. 508 // 509 // It is possible that version incompatibilities may result in a stored echo 510 // path of the incorrect size. In this case, the stored path should be 511 // discarded. 512 virtual int SetEchoPath(const void* echo_path, size_t size_bytes) = 0; 513 virtual int GetEchoPath(void* echo_path, size_t size_bytes) const = 0; 514 515 // The returned path size is guaranteed not to change for the lifetime of 516 // the application. 517 static size_t echo_path_size_bytes(); 518 519 protected: 520 virtual ~EchoControlMobile() {} 521 }; 522 523 // The automatic gain control (AGC) component brings the signal to an 524 // appropriate range. This is done by applying a digital gain directly and, in 525 // the analog mode, prescribing an analog gain to be applied at the audio HAL. 526 // 527 // Recommended to be enabled on the client-side. 528 class GainControl { 529 public: 530 virtual int Enable(bool enable) = 0; 531 virtual bool is_enabled() const = 0; 532 533 // When an analog mode is set, this must be called prior to |ProcessStream()| 534 // to pass the current analog level from the audio HAL. Must be within the 535 // range provided to |set_analog_level_limits()|. 536 virtual int set_stream_analog_level(int level) = 0; 537 538 // When an analog mode is set, this should be called after |ProcessStream()| 539 // to obtain the recommended new analog level for the audio HAL. It is the 540 // users responsibility to apply this level. 541 virtual int stream_analog_level() = 0; 542 543 enum Mode { 544 // Adaptive mode intended for use if an analog volume control is available 545 // on the capture device. It will require the user to provide coupling 546 // between the OS mixer controls and AGC through the |stream_analog_level()| 547 // functions. 548 // 549 // It consists of an analog gain prescription for the audio device and a 550 // digital compression stage. 551 kAdaptiveAnalog, 552 553 // Adaptive mode intended for situations in which an analog volume control 554 // is unavailable. It operates in a similar fashion to the adaptive analog 555 // mode, but with scaling instead applied in the digital domain. As with 556 // the analog mode, it additionally uses a digital compression stage. 557 kAdaptiveDigital, 558 559 // Fixed mode which enables only the digital compression stage also used by 560 // the two adaptive modes. 561 // 562 // It is distinguished from the adaptive modes by considering only a 563 // short time-window of the input signal. It applies a fixed gain through 564 // most of the input level range, and compresses (gradually reduces gain 565 // with increasing level) the input signal at higher levels. This mode is 566 // preferred on embedded devices where the capture signal level is 567 // predictable, so that a known gain can be applied. 568 kFixedDigital 569 }; 570 571 virtual int set_mode(Mode mode) = 0; 572 virtual Mode mode() const = 0; 573 574 // Sets the target peak |level| (or envelope) of the AGC in dBFs (decibels 575 // from digital full-scale). The convention is to use positive values. For 576 // instance, passing in a value of 3 corresponds to -3 dBFs, or a target 577 // level 3 dB below full-scale. Limited to [0, 31]. 578 // 579 // TODO(ajm): use a negative value here instead, if/when VoE will similarly 580 // update its interface. 581 virtual int set_target_level_dbfs(int level) = 0; 582 virtual int target_level_dbfs() const = 0; 583 584 // Sets the maximum |gain| the digital compression stage may apply, in dB. A 585 // higher number corresponds to greater compression, while a value of 0 will 586 // leave the signal uncompressed. Limited to [0, 90]. 587 virtual int set_compression_gain_db(int gain) = 0; 588 virtual int compression_gain_db() const = 0; 589 590 // When enabled, the compression stage will hard limit the signal to the 591 // target level. Otherwise, the signal will be compressed but not limited 592 // above the target level. 593 virtual int enable_limiter(bool enable) = 0; 594 virtual bool is_limiter_enabled() const = 0; 595 596 // Sets the |minimum| and |maximum| analog levels of the audio capture device. 597 // Must be set if and only if an analog mode is used. Limited to [0, 65535]. 598 virtual int set_analog_level_limits(int minimum, 599 int maximum) = 0; 600 virtual int analog_level_minimum() const = 0; 601 virtual int analog_level_maximum() const = 0; 602 603 // Returns true if the AGC has detected a saturation event (period where the 604 // signal reaches digital full-scale) in the current frame and the analog 605 // level cannot be reduced. 606 // 607 // This could be used as an indicator to reduce or disable analog mic gain at 608 // the audio HAL. 609 virtual bool stream_is_saturated() const = 0; 610 611 protected: 612 virtual ~GainControl() {} 613 }; 614 615 // A filtering component which removes DC offset and low-frequency noise. 616 // Recommended to be enabled on the client-side. 617 class HighPassFilter { 618 public: 619 virtual int Enable(bool enable) = 0; 620 virtual bool is_enabled() const = 0; 621 622 protected: 623 virtual ~HighPassFilter() {} 624 }; 625 626 // An estimation component used to retrieve level metrics. 627 class LevelEstimator { 628 public: 629 virtual int Enable(bool enable) = 0; 630 virtual bool is_enabled() const = 0; 631 632 // Returns the root mean square (RMS) level in dBFs (decibels from digital 633 // full-scale), or alternately dBov. It is computed over all primary stream 634 // frames since the last call to RMS(). The returned value is positive but 635 // should be interpreted as negative. It is constrained to [0, 127]. 636 // 637 // The computation follows: https://tools.ietf.org/html/rfc6465 638 // with the intent that it can provide the RTP audio level indication. 639 // 640 // Frames passed to ProcessStream() with an |_energy| of zero are considered 641 // to have been muted. The RMS of the frame will be interpreted as -127. 642 virtual int RMS() = 0; 643 644 protected: 645 virtual ~LevelEstimator() {} 646 }; 647 648 // The noise suppression (NS) component attempts to remove noise while 649 // retaining speech. Recommended to be enabled on the client-side. 650 // 651 // Recommended to be enabled on the client-side. 652 class NoiseSuppression { 653 public: 654 virtual int Enable(bool enable) = 0; 655 virtual bool is_enabled() const = 0; 656 657 // Determines the aggressiveness of the suppression. Increasing the level 658 // will reduce the noise level at the expense of a higher speech distortion. 659 enum Level { 660 kLow, 661 kModerate, 662 kHigh, 663 kVeryHigh 664 }; 665 666 virtual int set_level(Level level) = 0; 667 virtual Level level() const = 0; 668 669 // Returns the internally computed prior speech probability of current frame 670 // averaged over output channels. This is not supported in fixed point, for 671 // which |kUnsupportedFunctionError| is returned. 672 virtual float speech_probability() const = 0; 673 674 protected: 675 virtual ~NoiseSuppression() {} 676 }; 677 678 // The voice activity detection (VAD) component analyzes the stream to 679 // determine if voice is present. A facility is also provided to pass in an 680 // external VAD decision. 681 // 682 // In addition to |stream_has_voice()| the VAD decision is provided through the 683 // |AudioFrame| passed to |ProcessStream()|. The |vad_activity_| member will be 684 // modified to reflect the current decision. 685 class VoiceDetection { 686 public: 687 virtual int Enable(bool enable) = 0; 688 virtual bool is_enabled() const = 0; 689 690 // Returns true if voice is detected in the current frame. Should be called 691 // after |ProcessStream()|. 692 virtual bool stream_has_voice() const = 0; 693 694 // Some of the APM functionality requires a VAD decision. In the case that 695 // a decision is externally available for the current frame, it can be passed 696 // in here, before |ProcessStream()| is called. 697 // 698 // VoiceDetection does _not_ need to be enabled to use this. If it happens to 699 // be enabled, detection will be skipped for any frame in which an external 700 // VAD decision is provided. 701 virtual int set_stream_has_voice(bool has_voice) = 0; 702 703 // Specifies the likelihood that a frame will be declared to contain voice. 704 // A higher value makes it more likely that speech will not be clipped, at 705 // the expense of more noise being detected as voice. 706 enum Likelihood { 707 kVeryLowLikelihood, 708 kLowLikelihood, 709 kModerateLikelihood, 710 kHighLikelihood 711 }; 712 713 virtual int set_likelihood(Likelihood likelihood) = 0; 714 virtual Likelihood likelihood() const = 0; 715 716 // Sets the |size| of the frames in ms on which the VAD will operate. Larger 717 // frames will improve detection accuracy, but reduce the frequency of 718 // updates. 719 // 720 // This does not impact the size of frames passed to |ProcessStream()|. 721 virtual int set_frame_size_ms(int size) = 0; 722 virtual int frame_size_ms() const = 0; 723 724 protected: 725 virtual ~VoiceDetection() {} 726 }; 727 } // namespace webrtc 728 729 #endif // WEBRTC_MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ 730