1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_ 6 #define CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_ 7 8 #include "base/atomicops.h" 9 #include "base/files/file.h" 10 #include "base/synchronization/lock.h" 11 #include "base/threading/thread_checker.h" 12 #include "base/time/time.h" 13 #include "content/common/content_export.h" 14 #include "content/renderer/media/aec_dump_message_filter.h" 15 #include "content/renderer/media/webrtc_audio_device_impl.h" 16 #include "media/base/audio_converter.h" 17 #include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h" 18 #include "third_party/webrtc/modules/audio_processing/include/audio_processing.h" 19 #include "third_party/webrtc/modules/interface/module_common_types.h" 20 21 namespace blink { 22 class WebMediaConstraints; 23 } 24 25 namespace media { 26 class AudioBus; 27 class AudioFifo; 28 class AudioParameters; 29 } // namespace media 30 31 namespace webrtc { 32 class AudioFrame; 33 class TypingDetection; 34 } 35 36 namespace content { 37 38 class RTCMediaConstraints; 39 40 using webrtc::AudioProcessorInterface; 41 42 // This class owns an object of webrtc::AudioProcessing which contains signal 43 // processing components like AGC, AEC and NS. It enables the components based 44 // on the getUserMedia constraints, processes the data and outputs it in a unit 45 // of 10 ms data chunk. 46 class CONTENT_EXPORT MediaStreamAudioProcessor : 47 NON_EXPORTED_BASE(public WebRtcPlayoutDataSource::Sink), 48 NON_EXPORTED_BASE(public AudioProcessorInterface), 49 NON_EXPORTED_BASE(public AecDumpMessageFilter::AecDumpDelegate) { 50 public: 51 // Returns false if |kDisableAudioTrackProcessing| is set to true, otherwise 52 // returns true. 53 static bool IsAudioTrackProcessingEnabled(); 54 55 // |playout_data_source| is used to register this class as a sink to the 56 // WebRtc playout data for processing AEC. If clients do not enable AEC, 57 // |playout_data_source| won't be used. 58 MediaStreamAudioProcessor(const blink::WebMediaConstraints& constraints, 59 int effects, 60 WebRtcPlayoutDataSource* playout_data_source); 61 62 // Called when format of the capture data has changed. 63 // Called on the main render thread. The caller is responsible for stopping 64 // the capture thread before calling this method. 65 // After this method, the capture thread will be changed to a new capture 66 // thread. 67 void OnCaptureFormatChanged(const media::AudioParameters& source_params); 68 69 // Pushes capture data in |audio_source| to the internal FIFO. 70 // Called on the capture audio thread. 71 void PushCaptureData(const media::AudioBus* audio_source); 72 73 // Processes a block of 10 ms data from the internal FIFO and outputs it via 74 // |out|. |out| is the address of the pointer that will be pointed to 75 // the post-processed data if the method is returning a true. The lifetime 76 // of the data represeted by |out| is guaranteed to outlive the method call. 77 // That also says *|out| won't change until this method is called again. 78 // |new_volume| receives the new microphone volume from the AGC. 79 // The new microphoen volume range is [0, 255], and the value will be 0 if 80 // the microphone volume should not be adjusted. 81 // Returns true if the internal FIFO has at least 10 ms data for processing, 82 // otherwise false. 83 // |capture_delay|, |volume| and |key_pressed| will be passed to 84 // webrtc::AudioProcessing to help processing the data. 85 // Called on the capture audio thread. 86 bool ProcessAndConsumeData(base::TimeDelta capture_delay, 87 int volume, 88 bool key_pressed, 89 int* new_volume, 90 int16** out); 91 92 // Stops the audio processor, no more AEC dump or render data after calling 93 // this method. 94 void Stop(); 95 96 // The audio format of the input to the processor. 97 const media::AudioParameters& InputFormat() const; 98 99 // The audio format of the output from the processor. 100 const media::AudioParameters& OutputFormat() const; 101 102 // Accessor to check if the audio processing is enabled or not. 103 bool has_audio_processing() const { return audio_processing_ != NULL; } 104 105 // AecDumpMessageFilter::AecDumpDelegate implementation. 106 // Called on the main render thread. 107 virtual void OnAecDumpFile( 108 const IPC::PlatformFileForTransit& file_handle) OVERRIDE; 109 virtual void OnDisableAecDump() OVERRIDE; 110 virtual void OnIpcClosing() OVERRIDE; 111 112 protected: 113 friend class base::RefCountedThreadSafe<MediaStreamAudioProcessor>; 114 virtual ~MediaStreamAudioProcessor(); 115 116 private: 117 friend class MediaStreamAudioProcessorTest; 118 119 class MediaStreamAudioConverter; 120 121 // WebRtcPlayoutDataSource::Sink implementation. 122 virtual void OnPlayoutData(media::AudioBus* audio_bus, 123 int sample_rate, 124 int audio_delay_milliseconds) OVERRIDE; 125 virtual void OnPlayoutDataSourceChanged() OVERRIDE; 126 127 // webrtc::AudioProcessorInterface implementation. 128 // This method is called on the libjingle thread. 129 virtual void GetStats(AudioProcessorStats* stats) OVERRIDE; 130 131 // Helper to initialize the WebRtc AudioProcessing. 132 void InitializeAudioProcessingModule( 133 const blink::WebMediaConstraints& constraints, int effects); 134 135 // Helper to initialize the capture converter. 136 void InitializeCaptureConverter(const media::AudioParameters& source_params); 137 138 // Helper to initialize the render converter. 139 void InitializeRenderConverterIfNeeded(int sample_rate, 140 int number_of_channels, 141 int frames_per_buffer); 142 143 // Called by ProcessAndConsumeData(). 144 // Returns the new microphone volume in the range of |0, 255]. 145 // When the volume does not need to be updated, it returns 0. 146 int ProcessData(webrtc::AudioFrame* audio_frame, 147 base::TimeDelta capture_delay, 148 int volume, 149 bool key_pressed); 150 151 // Cached value for the render delay latency. This member is accessed by 152 // both the capture audio thread and the render audio thread. 153 base::subtle::Atomic32 render_delay_ms_; 154 155 // webrtc::AudioProcessing module which does AEC, AGC, NS, HighPass filter, 156 // ..etc. 157 scoped_ptr<webrtc::AudioProcessing> audio_processing_; 158 159 // Converter used for the down-mixing and resampling of the capture data. 160 scoped_ptr<MediaStreamAudioConverter> capture_converter_; 161 162 // AudioFrame used to hold the output of |capture_converter_|. 163 webrtc::AudioFrame capture_frame_; 164 165 // Converter used for the down-mixing and resampling of the render data when 166 // the AEC is enabled. 167 scoped_ptr<MediaStreamAudioConverter> render_converter_; 168 169 // AudioFrame used to hold the output of |render_converter_|. 170 webrtc::AudioFrame render_frame_; 171 172 // Data bus to help converting interleaved data to an AudioBus. 173 scoped_ptr<media::AudioBus> render_data_bus_; 174 175 // Raw pointer to the WebRtcPlayoutDataSource, which is valid for the 176 // lifetime of RenderThread. 177 WebRtcPlayoutDataSource* playout_data_source_; 178 179 // Used to DCHECK that the destructor is called on the main render thread. 180 base::ThreadChecker main_thread_checker_; 181 182 // Used to DCHECK that some methods are called on the capture audio thread. 183 base::ThreadChecker capture_thread_checker_; 184 185 // Used to DCHECK that PushRenderData() is called on the render audio thread. 186 base::ThreadChecker render_thread_checker_; 187 188 // Flag to enable the stereo channels mirroring. 189 bool audio_mirroring_; 190 191 // Used by the typing detection. 192 scoped_ptr<webrtc::TypingDetection> typing_detector_; 193 194 // This flag is used to show the result of typing detection. 195 // It can be accessed by the capture audio thread and by the libjingle thread 196 // which calls GetStats(). 197 base::subtle::Atomic32 typing_detected_; 198 199 // Communication with browser for AEC dump. 200 scoped_refptr<AecDumpMessageFilter> aec_dump_message_filter_; 201 202 // Flag to avoid executing Stop() more than once. 203 bool stopped_; 204 }; 205 206 } // namespace content 207 208 #endif // CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_ 209