Home | History | Annotate | Download | only in media 
      1 // Copyright 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #ifndef CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_
      6 #define CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_
      7 
      8 #include "base/atomicops.h"
      9 #include "base/files/file.h"
     10 #include "base/synchronization/lock.h"
     11 #include "base/threading/thread_checker.h"
     12 #include "base/time/time.h"
     13 #include "content/common/content_export.h"
     14 #include "content/renderer/media/aec_dump_message_filter.h"
     15 #include "content/renderer/media/webrtc_audio_device_impl.h"
     16 #include "media/base/audio_converter.h"
     17 #include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h"
     18 #include "third_party/webrtc/modules/audio_processing/include/audio_processing.h"
     19 #include "third_party/webrtc/modules/interface/module_common_types.h"
     20 
     21 namespace blink {
     22 class WebMediaConstraints;
     23 }
     24 
     25 namespace media {
     26 class AudioBus;
     27 class AudioFifo;
     28 class AudioParameters;
     29 }  // namespace media
     30 
     31 namespace webrtc {
     32 class AudioFrame;
     33 class TypingDetection;
     34 }
     35 
     36 namespace content {
     37 
     38 class MediaStreamAudioBus;
     39 class MediaStreamAudioFifo;
     40 class RTCMediaConstraints;
     41 
     42 using webrtc::AudioProcessorInterface;
     43 
     44 // This class owns an object of webrtc::AudioProcessing which contains signal
     45 // processing components like AGC, AEC and NS. It enables the components based
     46 // on the getUserMedia constraints, processes the data and outputs it in a unit
     47 // of 10 ms data chunk.
     48 class CONTENT_EXPORT MediaStreamAudioProcessor :
     49     NON_EXPORTED_BASE(public WebRtcPlayoutDataSource::Sink),
     50     NON_EXPORTED_BASE(public AudioProcessorInterface),
     51     NON_EXPORTED_BASE(public AecDumpMessageFilter::AecDumpDelegate) {
     52  public:
     53   // Returns false if |kDisableAudioTrackProcessing| is set to true, otherwise
     54   // returns true.
     55   static bool IsAudioTrackProcessingEnabled();
     56 
     57   // |playout_data_source| is used to register this class as a sink to the
     58   // WebRtc playout data for processing AEC. If clients do not enable AEC,
     59   // |playout_data_source| won't be used.
     60   MediaStreamAudioProcessor(const blink::WebMediaConstraints& constraints,
     61                             int effects,
     62                             WebRtcPlayoutDataSource* playout_data_source);
     63 
     64   // Called when the format of the capture data has changed.
     65   // Called on the main render thread. The caller is responsible for stopping
     66   // the capture thread before calling this method.
     67   // After this method, the capture thread will be changed to a new capture
     68   // thread.
     69   void OnCaptureFormatChanged(const media::AudioParameters& source_params);
     70 
     71   // Pushes capture data in |audio_source| to the internal FIFO. Each call to
     72   // this method should be followed by calls to ProcessAndConsumeData() while
     73   // it returns false, to pull out all available data.
     74   // Called on the capture audio thread.
     75   void PushCaptureData(const media::AudioBus* audio_source);
     76 
     77   // Processes a block of 10 ms data from the internal FIFO and outputs it via
     78   // |out|. |out| is the address of the pointer that will be pointed to
     79   // the post-processed data if the method is returning a true. The lifetime
     80   // of the data represeted by |out| is guaranteed until this method is called
     81   // again.
     82   // |new_volume| receives the new microphone volume from the AGC.
     83   // The new microphone volume range is [0, 255], and the value will be 0 if
     84   // the microphone volume should not be adjusted.
     85   // Returns true if the internal FIFO has at least 10 ms data for processing,
     86   // otherwise false.
     87   // Called on the capture audio thread.
     88   //
     89   // TODO(ajm): Don't we want this to output float?
     90   bool ProcessAndConsumeData(base::TimeDelta capture_delay,
     91                              int volume,
     92                              bool key_pressed,
     93                              int* new_volume,
     94                              int16** out);
     95 
     96   // Stops the audio processor, no more AEC dump or render data after calling
     97   // this method.
     98   void Stop();
     99 
    100   // The audio formats of the capture input to and output from the processor.
    101   // Must only be called on the main render or audio capture threads.
    102   const media::AudioParameters& InputFormat() const;
    103   const media::AudioParameters& OutputFormat() const;
    104 
    105   // Accessor to check if the audio processing is enabled or not.
    106   bool has_audio_processing() const { return audio_processing_ != NULL; }
    107 
    108   // AecDumpMessageFilter::AecDumpDelegate implementation.
    109   // Called on the main render thread.
    110   virtual void OnAecDumpFile(
    111       const IPC::PlatformFileForTransit& file_handle) OVERRIDE;
    112   virtual void OnDisableAecDump() OVERRIDE;
    113   virtual void OnIpcClosing() OVERRIDE;
    114 
    115  protected:
    116   friend class base::RefCountedThreadSafe<MediaStreamAudioProcessor>;
    117   virtual ~MediaStreamAudioProcessor();
    118 
    119  private:
    120   friend class MediaStreamAudioProcessorTest;
    121   FRIEND_TEST_ALL_PREFIXES(MediaStreamAudioProcessorTest,
    122                            GetAecDumpMessageFilter);
    123 
    124   // WebRtcPlayoutDataSource::Sink implementation.
    125   virtual void OnPlayoutData(media::AudioBus* audio_bus,
    126                              int sample_rate,
    127                              int audio_delay_milliseconds) OVERRIDE;
    128   virtual void OnPlayoutDataSourceChanged() OVERRIDE;
    129 
    130   // webrtc::AudioProcessorInterface implementation.
    131   // This method is called on the libjingle thread.
    132   virtual void GetStats(AudioProcessorStats* stats) OVERRIDE;
    133 
    134   // Helper to initialize the WebRtc AudioProcessing.
    135   void InitializeAudioProcessingModule(
    136       const blink::WebMediaConstraints& constraints, int effects);
    137 
    138   // Helper to initialize the capture converter.
    139   void InitializeCaptureFifo(const media::AudioParameters& input_format);
    140 
    141   // Helper to initialize the render converter.
    142   void InitializeRenderFifoIfNeeded(int sample_rate,
    143                                     int number_of_channels,
    144                                     int frames_per_buffer);
    145 
    146   // Called by ProcessAndConsumeData().
    147   // Returns the new microphone volume in the range of |0, 255].
    148   // When the volume does not need to be updated, it returns 0.
    149   int ProcessData(const float* const* process_ptrs,
    150                   int process_frames,
    151                   base::TimeDelta capture_delay,
    152                   int volume,
    153                   bool key_pressed,
    154                   float* const* output_ptrs);
    155 
    156   // Cached value for the render delay latency. This member is accessed by
    157   // both the capture audio thread and the render audio thread.
    158   base::subtle::Atomic32 render_delay_ms_;
    159 
    160   // Module to handle processing and format conversion.
    161   scoped_ptr<webrtc::AudioProcessing> audio_processing_;
    162 
    163   // FIFO to provide 10 ms capture chunks.
    164   scoped_ptr<MediaStreamAudioFifo> capture_fifo_;
    165   // Receives processing output.
    166   scoped_ptr<MediaStreamAudioBus> output_bus_;
    167   // Receives interleaved int16 data for output.
    168   scoped_ptr<int16[]> output_data_;
    169 
    170   // FIFO to provide 10 ms render chunks when the AEC is enabled.
    171   scoped_ptr<MediaStreamAudioFifo> render_fifo_;
    172 
    173   // These are mutated on the main render thread in OnCaptureFormatChanged().
    174   // The caller guarantees this does not run concurrently with accesses on the
    175   // capture audio thread.
    176   media::AudioParameters input_format_;
    177   media::AudioParameters output_format_;
    178   // Only used on the render audio thread.
    179   media::AudioParameters render_format_;
    180 
    181   // Raw pointer to the WebRtcPlayoutDataSource, which is valid for the
    182   // lifetime of RenderThread.
    183   WebRtcPlayoutDataSource* playout_data_source_;
    184 
    185   // Used to DCHECK that some methods are called on the main render thread.
    186   base::ThreadChecker main_thread_checker_;
    187   // Used to DCHECK that some methods are called on the capture audio thread.
    188   base::ThreadChecker capture_thread_checker_;
    189   // Used to DCHECK that some methods are called on the render audio thread.
    190   base::ThreadChecker render_thread_checker_;
    191 
    192   // Flag to enable stereo channel mirroring.
    193   bool audio_mirroring_;
    194 
    195   scoped_ptr<webrtc::TypingDetection> typing_detector_;
    196   // This flag is used to show the result of typing detection.
    197   // It can be accessed by the capture audio thread and by the libjingle thread
    198   // which calls GetStats().
    199   base::subtle::Atomic32 typing_detected_;
    200 
    201   // Communication with browser for AEC dump.
    202   scoped_refptr<AecDumpMessageFilter> aec_dump_message_filter_;
    203 
    204   // Flag to avoid executing Stop() more than once.
    205   bool stopped_;
    206 };
    207 
    208 }  // namespace content
    209 
    210 #endif  // CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_
    211