Home | History | Annotate | Download | only in base 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 //
      5 // AudioConverter implementation.  Uses MultiChannelSincResampler for resampling
      6 // audio, ChannelMixer for channel mixing, and AudioPullFifo for buffering.
      7 //
      8 // Delay estimates are provided to InputCallbacks based on the frame delay
      9 // information reported via the resampler and FIFO units.
     10 
     11 #include "media/base/audio_converter.h"
     12 
     13 #include <algorithm>
     14 
     15 #include "base/bind.h"
     16 #include "base/bind_helpers.h"
     17 #include "media/base/audio_bus.h"
     18 #include "media/base/audio_pull_fifo.h"
     19 #include "media/base/channel_mixer.h"
     20 #include "media/base/multi_channel_resampler.h"
     21 #include "media/base/vector_math.h"
     22 
     23 namespace media {
     24 
     25 AudioConverter::AudioConverter(const AudioParameters& input_params,
     26                                const AudioParameters& output_params,
     27                                bool disable_fifo)
     28     : chunk_size_(input_params.frames_per_buffer()),
     29       downmix_early_(false),
     30       resampler_frame_delay_(0),
     31       input_channel_count_(input_params.channels()) {
     32   CHECK(input_params.IsValid());
     33   CHECK(output_params.IsValid());
     34 
     35   // Handle different input and output channel layouts.
     36   if (input_params.channel_layout() != output_params.channel_layout()) {
     37     DVLOG(1) << "Remixing channel layout from " << input_params.channel_layout()
     38              << " to " << output_params.channel_layout() << "; from "
     39              << input_params.channels() << " channels to "
     40              << output_params.channels() << " channels.";
     41     channel_mixer_.reset(new ChannelMixer(input_params, output_params));
     42 
     43     // Pare off data as early as we can for efficiency.
     44     downmix_early_ = input_params.channels() > output_params.channels();
     45   }
     46 
     47   // Only resample if necessary since it's expensive.
     48   if (input_params.sample_rate() != output_params.sample_rate()) {
     49     DVLOG(1) << "Resampling from " << input_params.sample_rate() << " to "
     50              << output_params.sample_rate();
     51     const int request_size = disable_fifo ? SincResampler::kDefaultRequestSize :
     52         input_params.frames_per_buffer();
     53     const double io_sample_rate_ratio =
     54         input_params.sample_rate() /
     55         static_cast<double>(output_params.sample_rate());
     56     resampler_.reset(new MultiChannelResampler(
     57         downmix_early_ ? output_params.channels() : input_params.channels(),
     58         io_sample_rate_ratio,
     59         request_size,
     60         base::Bind(&AudioConverter::ProvideInput, base::Unretained(this))));
     61   }
     62 
     63   input_frame_duration_ = base::TimeDelta::FromMicroseconds(
     64       base::Time::kMicrosecondsPerSecond /
     65       static_cast<double>(input_params.sample_rate()));
     66   output_frame_duration_ = base::TimeDelta::FromMicroseconds(
     67       base::Time::kMicrosecondsPerSecond /
     68       static_cast<double>(output_params.sample_rate()));
     69 
     70   // The resampler can be configured to work with a specific request size, so a
     71   // FIFO is not necessary when resampling.
     72   if (disable_fifo || resampler_)
     73     return;
     74 
     75   // Since the output device may want a different buffer size than the caller
     76   // asked for, we need to use a FIFO to ensure that both sides read in chunk
     77   // sizes they're configured for.
     78   if (input_params.frames_per_buffer() != output_params.frames_per_buffer()) {
     79     DVLOG(1) << "Rebuffering from " << input_params.frames_per_buffer()
     80              << " to " << output_params.frames_per_buffer();
     81     chunk_size_ = input_params.frames_per_buffer();
     82     audio_fifo_.reset(new AudioPullFifo(
     83         downmix_early_ ? output_params.channels() : input_params.channels(),
     84         chunk_size_,
     85         base::Bind(&AudioConverter::SourceCallback, base::Unretained(this))));
     86   }
     87 }
     88 
     89 AudioConverter::~AudioConverter() {}
     90 
     91 void AudioConverter::AddInput(InputCallback* input) {
     92   DCHECK(std::find(transform_inputs_.begin(), transform_inputs_.end(), input) ==
     93          transform_inputs_.end());
     94   transform_inputs_.push_back(input);
     95 }
     96 
     97 void AudioConverter::RemoveInput(InputCallback* input) {
     98   DCHECK(std::find(transform_inputs_.begin(), transform_inputs_.end(), input) !=
     99          transform_inputs_.end());
    100   transform_inputs_.remove(input);
    101 
    102   if (transform_inputs_.empty())
    103     Reset();
    104 }
    105 
    106 void AudioConverter::Reset() {
    107   if (audio_fifo_)
    108     audio_fifo_->Clear();
    109   if (resampler_)
    110     resampler_->Flush();
    111 }
    112 
    113 int AudioConverter::ChunkSize() const {
    114   if (!resampler_)
    115     return chunk_size_;
    116   return resampler_->ChunkSize();
    117 }
    118 
    119 void AudioConverter::ConvertWithDelay(const base::TimeDelta& initial_delay,
    120                                       AudioBus* dest) {
    121   initial_delay_ = initial_delay;
    122 
    123   if (transform_inputs_.empty()) {
    124     dest->Zero();
    125     return;
    126   }
    127 
    128   // Determine if channel mixing should be done and if it should be done before
    129   // or after resampling.  If it's possible to reduce the channel count prior to
    130   // resampling we can save a lot of processing time.  Vice versa, we don't want
    131   // to increase the channel count prior to resampling for the same reason.
    132   bool needs_mixing = channel_mixer_ && !downmix_early_;
    133 
    134   if (needs_mixing)
    135     CreateUnmixedAudioIfNecessary(dest->frames());
    136 
    137   AudioBus* temp_dest = needs_mixing ? unmixed_audio_.get() : dest;
    138   DCHECK(temp_dest);
    139 
    140   // Figure out which method to call based on whether we're resampling and
    141   // rebuffering, just resampling, or just mixing.  We want to avoid any extra
    142   // steps when possible since we may be converting audio data in real time.
    143   if (!resampler_ && !audio_fifo_) {
    144     SourceCallback(0, temp_dest);
    145   } else {
    146     if (resampler_)
    147       resampler_->Resample(temp_dest->frames(), temp_dest);
    148     else
    149       ProvideInput(0, temp_dest);
    150   }
    151 
    152   // Finally upmix the channels if we didn't do so earlier.
    153   if (needs_mixing) {
    154     DCHECK_EQ(temp_dest->frames(), dest->frames());
    155     channel_mixer_->Transform(temp_dest, dest);
    156   }
    157 }
    158 
    159 void AudioConverter::Convert(AudioBus* dest) {
    160   ConvertWithDelay(base::TimeDelta::FromMilliseconds(0), dest);
    161 }
    162 
    163 void AudioConverter::SourceCallback(int fifo_frame_delay, AudioBus* dest) {
    164   const bool needs_downmix = channel_mixer_ && downmix_early_;
    165 
    166   if (!mixer_input_audio_bus_ ||
    167       mixer_input_audio_bus_->frames() != dest->frames()) {
    168     mixer_input_audio_bus_ =
    169         AudioBus::Create(input_channel_count_, dest->frames());
    170   }
    171 
    172   // If we're downmixing early we need a temporary AudioBus which matches
    173   // the the input channel count and input frame size since we're passing
    174   // |unmixed_audio_| directly to the |source_callback_|.
    175   if (needs_downmix)
    176     CreateUnmixedAudioIfNecessary(dest->frames());
    177 
    178   AudioBus* const temp_dest = needs_downmix ? unmixed_audio_.get() : dest;
    179 
    180   // Sanity check our inputs.
    181   DCHECK_EQ(temp_dest->frames(), mixer_input_audio_bus_->frames());
    182   DCHECK_EQ(temp_dest->channels(), mixer_input_audio_bus_->channels());
    183 
    184   // Calculate the buffer delay for this callback.
    185   base::TimeDelta buffer_delay = initial_delay_;
    186   if (resampler_) {
    187     buffer_delay += base::TimeDelta::FromMicroseconds(
    188         resampler_frame_delay_ * output_frame_duration_.InMicroseconds());
    189   }
    190   if (audio_fifo_) {
    191     buffer_delay += base::TimeDelta::FromMicroseconds(
    192         fifo_frame_delay * input_frame_duration_.InMicroseconds());
    193   }
    194 
    195   // If we only have a single input, avoid an extra copy.
    196   AudioBus* const provide_input_dest =
    197       transform_inputs_.size() == 1 ? temp_dest : mixer_input_audio_bus_.get();
    198 
    199   // Have each mixer render its data into an output buffer then mix the result.
    200   for (auto* input : transform_inputs_) {
    201     const float volume = input->ProvideInput(provide_input_dest, buffer_delay);
    202 
    203     // Optimize the most common single input, full volume case.
    204     if (input == transform_inputs_.front()) {
    205       if (volume == 1.0f) {
    206         if (temp_dest != provide_input_dest)
    207           provide_input_dest->CopyTo(temp_dest);
    208       } else if (volume > 0) {
    209         for (int i = 0; i < provide_input_dest->channels(); ++i) {
    210           vector_math::FMUL(
    211               provide_input_dest->channel(i), volume,
    212               provide_input_dest->frames(), temp_dest->channel(i));
    213         }
    214       } else {
    215         // Zero |temp_dest| otherwise, so we're mixing into a clean buffer.
    216         temp_dest->Zero();
    217       }
    218 
    219       continue;
    220     }
    221 
    222     // Volume adjust and mix each mixer input into |temp_dest| after rendering.
    223     if (volume > 0) {
    224       for (int i = 0; i < mixer_input_audio_bus_->channels(); ++i) {
    225         vector_math::FMAC(
    226             mixer_input_audio_bus_->channel(i), volume,
    227             mixer_input_audio_bus_->frames(), temp_dest->channel(i));
    228       }
    229     }
    230   }
    231 
    232   if (needs_downmix) {
    233     DCHECK_EQ(temp_dest->frames(), dest->frames());
    234     channel_mixer_->Transform(temp_dest, dest);
    235   }
    236 }
    237 
    238 void AudioConverter::ProvideInput(int resampler_frame_delay, AudioBus* dest) {
    239   resampler_frame_delay_ = resampler_frame_delay;
    240   if (audio_fifo_)
    241     audio_fifo_->Consume(dest, dest->frames());
    242   else
    243     SourceCallback(0, dest);
    244 }
    245 
    246 void AudioConverter::CreateUnmixedAudioIfNecessary(int frames) {
    247   if (!unmixed_audio_ || unmixed_audio_->frames() != frames)
    248     unmixed_audio_ = AudioBus::Create(input_channel_count_, frames);
    249 }
    250 
    251 }  // namespace media
    252