Home | History | Annotate | Download | only in audio_processing 
      1 /*
      2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
     12 #define WEBRTC_MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
     13 
     14 #include <list>
     15 #include <string>
     16 #include <vector>
     17 
     18 #include "webrtc/base/criticalsection.h"
     19 #include "webrtc/base/scoped_ptr.h"
     20 #include "webrtc/base/thread_annotations.h"
     21 #include "webrtc/modules/audio_processing/audio_buffer.h"
     22 #include "webrtc/modules/audio_processing/include/audio_processing.h"
     23 #include "webrtc/system_wrappers/include/file_wrapper.h"
     24 
     25 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
     26 // Files generated at build-time by the protobuf compiler.
     27 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD
     28 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h"
     29 #else
     30 #include "webrtc/audio_processing/debug.pb.h"
     31 #endif
     32 #endif  // WEBRTC_AUDIOPROC_DEBUG_DUMP
     33 
     34 namespace webrtc {
     35 
     36 class AgcManagerDirect;
     37 class AudioConverter;
     38 
     39 template<typename T>
     40 class Beamformer;
     41 
     42 class AudioProcessingImpl : public AudioProcessing {
     43  public:
     44   // Methods forcing APM to run in a single-threaded manner.
     45   // Acquires both the render and capture locks.
     46   explicit AudioProcessingImpl(const Config& config);
     47   // AudioProcessingImpl takes ownership of beamformer.
     48   AudioProcessingImpl(const Config& config, Beamformer<float>* beamformer);
     49   virtual ~AudioProcessingImpl();
     50   int Initialize() override;
     51   int Initialize(int input_sample_rate_hz,
     52                  int output_sample_rate_hz,
     53                  int reverse_sample_rate_hz,
     54                  ChannelLayout input_layout,
     55                  ChannelLayout output_layout,
     56                  ChannelLayout reverse_layout) override;
     57   int Initialize(const ProcessingConfig& processing_config) override;
     58   void SetExtraOptions(const Config& config) override;
     59   void UpdateHistogramsOnCallEnd() override;
     60   int StartDebugRecording(const char filename[kMaxFilenameSize]) override;
     61   int StartDebugRecording(FILE* handle) override;
     62   int StartDebugRecordingForPlatformFile(rtc::PlatformFile handle) override;
     63   int StopDebugRecording() override;
     64 
     65   // Capture-side exclusive methods possibly running APM in a
     66   // multi-threaded manner. Acquire the capture lock.
     67   int ProcessStream(AudioFrame* frame) override;
     68   int ProcessStream(const float* const* src,
     69                     size_t samples_per_channel,
     70                     int input_sample_rate_hz,
     71                     ChannelLayout input_layout,
     72                     int output_sample_rate_hz,
     73                     ChannelLayout output_layout,
     74                     float* const* dest) override;
     75   int ProcessStream(const float* const* src,
     76                     const StreamConfig& input_config,
     77                     const StreamConfig& output_config,
     78                     float* const* dest) override;
     79   void set_output_will_be_muted(bool muted) override;
     80   int set_stream_delay_ms(int delay) override;
     81   void set_delay_offset_ms(int offset) override;
     82   int delay_offset_ms() const override;
     83   void set_stream_key_pressed(bool key_pressed) override;
     84   int input_sample_rate_hz() const override;
     85 
     86   // Render-side exclusive methods possibly running APM in a
     87   // multi-threaded manner. Acquire the render lock.
     88   int AnalyzeReverseStream(AudioFrame* frame) override;
     89   int ProcessReverseStream(AudioFrame* frame) override;
     90   int AnalyzeReverseStream(const float* const* data,
     91                            size_t samples_per_channel,
     92                            int sample_rate_hz,
     93                            ChannelLayout layout) override;
     94   int ProcessReverseStream(const float* const* src,
     95                            const StreamConfig& reverse_input_config,
     96                            const StreamConfig& reverse_output_config,
     97                            float* const* dest) override;
     98 
     99   // Methods only accessed from APM submodules or
    100   // from AudioProcessing tests in a single-threaded manner.
    101   // Hence there is no need for locks in these.
    102   int proc_sample_rate_hz() const override;
    103   int proc_split_sample_rate_hz() const override;
    104   size_t num_input_channels() const override;
    105   size_t num_proc_channels() const override;
    106   size_t num_output_channels() const override;
    107   size_t num_reverse_channels() const override;
    108   int stream_delay_ms() const override;
    109   bool was_stream_delay_set() const override
    110       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
    111 
    112   // Methods returning pointers to APM submodules.
    113   // No locks are aquired in those, as those locks
    114   // would offer no protection (the submodules are
    115   // created only once in a single-treaded manner
    116   // during APM creation).
    117   EchoCancellation* echo_cancellation() const override;
    118   EchoControlMobile* echo_control_mobile() const override;
    119   GainControl* gain_control() const override;
    120   HighPassFilter* high_pass_filter() const override;
    121   LevelEstimator* level_estimator() const override;
    122   NoiseSuppression* noise_suppression() const override;
    123   VoiceDetection* voice_detection() const override;
    124 
    125  protected:
    126   // Overridden in a mock.
    127   virtual int InitializeLocked()
    128       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
    129 
    130  private:
    131   struct ApmPublicSubmodules;
    132   struct ApmPrivateSubmodules;
    133 
    134 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
    135   // State for the debug dump.
    136   struct ApmDebugDumpThreadState {
    137     ApmDebugDumpThreadState() : event_msg(new audioproc::Event()) {}
    138     rtc::scoped_ptr<audioproc::Event> event_msg;  // Protobuf message.
    139     std::string event_str;  // Memory for protobuf serialization.
    140 
    141     // Serialized string of last saved APM configuration.
    142     std::string last_serialized_config;
    143   };
    144 
    145   struct ApmDebugDumpState {
    146     ApmDebugDumpState() : debug_file(FileWrapper::Create()) {}
    147     rtc::scoped_ptr<FileWrapper> debug_file;
    148     ApmDebugDumpThreadState render;
    149     ApmDebugDumpThreadState capture;
    150   };
    151 #endif
    152 
    153   // Method for modifying the formats struct that are called from both
    154   // the render and capture threads. The check for whether modifications
    155   // are needed is done while holding the render lock only, thereby avoiding
    156   // that the capture thread blocks the render thread.
    157   // The struct is modified in a single-threaded manner by holding both the
    158   // render and capture locks.
    159   int MaybeInitialize(const ProcessingConfig& config)
    160       EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
    161 
    162   int MaybeInitializeRender(const ProcessingConfig& processing_config)
    163       EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
    164 
    165   int MaybeInitializeCapture(const ProcessingConfig& processing_config)
    166       EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
    167 
    168   // Method for checking for the need of conversion. Accesses the formats
    169   // structs in a read manner but the requirement for the render lock to be held
    170   // was added as it currently anyway is always called in that manner.
    171   bool rev_conversion_needed() const EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
    172   bool render_check_rev_conversion_needed() const
    173       EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
    174 
    175   // Methods requiring APM running in a single-threaded manner.
    176   // Are called with both the render and capture locks already
    177   // acquired.
    178   void InitializeExperimentalAgc()
    179       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
    180   void InitializeTransient()
    181       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
    182   void InitializeBeamformer()
    183       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
    184   void InitializeIntelligibility()
    185       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
    186   void InitializeHighPassFilter()
    187       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
    188   void InitializeNoiseSuppression()
    189       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
    190   void InitializeLevelEstimator()
    191       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
    192   void InitializeVoiceDetection()
    193       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
    194   int InitializeLocked(const ProcessingConfig& config)
    195       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
    196 
    197   // Capture-side exclusive methods possibly running APM in a multi-threaded
    198   // manner that are called with the render lock already acquired.
    199   int ProcessStreamLocked() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
    200   bool output_copy_needed(bool is_data_processed) const
    201       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
    202   bool is_data_processed() const EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
    203   bool synthesis_needed(bool is_data_processed) const
    204       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
    205   bool analysis_needed(bool is_data_processed) const
    206       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
    207   void MaybeUpdateHistograms() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
    208 
    209   // Render-side exclusive methods possibly running APM in a multi-threaded
    210   // manner that are called with the render lock already acquired.
    211   // TODO(ekm): Remove once all clients updated to new interface.
    212   int AnalyzeReverseStreamLocked(const float* const* src,
    213                                  const StreamConfig& input_config,
    214                                  const StreamConfig& output_config)
    215       EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
    216   bool is_rev_processed() const EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
    217   int ProcessReverseStreamLocked() EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
    218 
    219 // Debug dump methods that are internal and called without locks.
    220 // TODO(peah): Make thread safe.
    221 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
    222   // TODO(andrew): make this more graceful. Ideally we would split this stuff
    223   // out into a separate class with an "enabled" and "disabled" implementation.
    224   static int WriteMessageToDebugFile(FileWrapper* debug_file,
    225                                      rtc::CriticalSection* crit_debug,
    226                                      ApmDebugDumpThreadState* debug_state);
    227   int WriteInitMessage() EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
    228 
    229   // Writes Config message. If not |forced|, only writes the current config if
    230   // it is different from the last saved one; if |forced|, writes the config
    231   // regardless of the last saved.
    232   int WriteConfigMessage(bool forced) EXCLUSIVE_LOCKS_REQUIRED(crit_capture_)
    233       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
    234 
    235   // Critical section.
    236   mutable rtc::CriticalSection crit_debug_;
    237 
    238   // Debug dump state.
    239   ApmDebugDumpState debug_dump_;
    240 #endif
    241 
    242   // Critical sections.
    243   mutable rtc::CriticalSection crit_render_ ACQUIRED_BEFORE(crit_capture_);
    244   mutable rtc::CriticalSection crit_capture_;
    245 
    246   // Structs containing the pointers to the submodules.
    247   rtc::scoped_ptr<ApmPublicSubmodules> public_submodules_;
    248   rtc::scoped_ptr<ApmPrivateSubmodules> private_submodules_
    249       GUARDED_BY(crit_capture_);
    250 
    251   // State that is written to while holding both the render and capture locks
    252   // but can be read without any lock being held.
    253   // As this is only accessed internally of APM, and all internal methods in APM
    254   // either are holding the render or capture locks, this construct is safe as
    255   // it is not possible to read the variables while writing them.
    256   struct ApmFormatState {
    257     ApmFormatState()
    258         :  // Format of processing streams at input/output call sites.
    259           api_format({{{kSampleRate16kHz, 1, false},
    260                        {kSampleRate16kHz, 1, false},
    261                        {kSampleRate16kHz, 1, false},
    262                        {kSampleRate16kHz, 1, false}}}),
    263           rev_proc_format(kSampleRate16kHz, 1) {}
    264     ProcessingConfig api_format;
    265     StreamConfig rev_proc_format;
    266   } formats_;
    267 
    268   // APM constants.
    269   const struct ApmConstants {
    270     ApmConstants(int agc_startup_min_volume,
    271                  bool use_new_agc,
    272                  bool intelligibility_enabled)
    273         :  // Format of processing streams at input/output call sites.
    274           agc_startup_min_volume(agc_startup_min_volume),
    275           use_new_agc(use_new_agc),
    276           intelligibility_enabled(intelligibility_enabled) {}
    277     int agc_startup_min_volume;
    278     bool use_new_agc;
    279     bool intelligibility_enabled;
    280   } constants_;
    281 
    282   struct ApmCaptureState {
    283     ApmCaptureState(bool transient_suppressor_enabled,
    284                     const std::vector<Point>& array_geometry,
    285                     SphericalPointf target_direction)
    286         : aec_system_delay_jumps(-1),
    287           delay_offset_ms(0),
    288           was_stream_delay_set(false),
    289           last_stream_delay_ms(0),
    290           last_aec_system_delay_ms(0),
    291           stream_delay_jumps(-1),
    292           output_will_be_muted(false),
    293           key_pressed(false),
    294           transient_suppressor_enabled(transient_suppressor_enabled),
    295           array_geometry(array_geometry),
    296           target_direction(target_direction),
    297           fwd_proc_format(kSampleRate16kHz),
    298           split_rate(kSampleRate16kHz) {}
    299     int aec_system_delay_jumps;
    300     int delay_offset_ms;
    301     bool was_stream_delay_set;
    302     int last_stream_delay_ms;
    303     int last_aec_system_delay_ms;
    304     int stream_delay_jumps;
    305     bool output_will_be_muted;
    306     bool key_pressed;
    307     bool transient_suppressor_enabled;
    308     std::vector<Point> array_geometry;
    309     SphericalPointf target_direction;
    310     rtc::scoped_ptr<AudioBuffer> capture_audio;
    311     // Only the rate and samples fields of fwd_proc_format_ are used because the
    312     // forward processing number of channels is mutable and is tracked by the
    313     // capture_audio_.
    314     StreamConfig fwd_proc_format;
    315     int split_rate;
    316   } capture_ GUARDED_BY(crit_capture_);
    317 
    318   struct ApmCaptureNonLockedState {
    319     ApmCaptureNonLockedState(bool beamformer_enabled)
    320         : fwd_proc_format(kSampleRate16kHz),
    321           split_rate(kSampleRate16kHz),
    322           stream_delay_ms(0),
    323           beamformer_enabled(beamformer_enabled) {}
    324     // Only the rate and samples fields of fwd_proc_format_ are used because the
    325     // forward processing number of channels is mutable and is tracked by the
    326     // capture_audio_.
    327     StreamConfig fwd_proc_format;
    328     int split_rate;
    329     int stream_delay_ms;
    330     bool beamformer_enabled;
    331   } capture_nonlocked_;
    332 
    333   struct ApmRenderState {
    334     rtc::scoped_ptr<AudioConverter> render_converter;
    335     rtc::scoped_ptr<AudioBuffer> render_audio;
    336   } render_ GUARDED_BY(crit_render_);
    337 };
    338 
    339 }  // namespace webrtc
    340 
    341 #endif  // WEBRTC_MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
    342