1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 // 5 // AudioConverter implementation. Uses MultiChannelSincResampler for resampling 6 // audio, ChannelMixer for channel mixing, and AudioPullFifo for buffering. 7 // 8 // Delay estimates are provided to InputCallbacks based on the frame delay 9 // information reported via the resampler and FIFO units. 10 11 #include "media/base/audio_converter.h" 12 13 #include <algorithm> 14 15 #include "base/bind.h" 16 #include "base/bind_helpers.h" 17 #include "media/base/audio_bus.h" 18 #include "media/base/audio_pull_fifo.h" 19 #include "media/base/channel_mixer.h" 20 #include "media/base/multi_channel_resampler.h" 21 #include "media/base/vector_math.h" 22 23 namespace media { 24 25 AudioConverter::AudioConverter(const AudioParameters& input_params, 26 const AudioParameters& output_params, 27 bool disable_fifo) 28 : chunk_size_(input_params.frames_per_buffer()), 29 downmix_early_(false), 30 resampler_frame_delay_(0), 31 input_channel_count_(input_params.channels()) { 32 CHECK(input_params.IsValid()); 33 CHECK(output_params.IsValid()); 34 35 // Handle different input and output channel layouts. 36 if (input_params.channel_layout() != output_params.channel_layout()) { 37 DVLOG(1) << "Remixing channel layout from " << input_params.channel_layout() 38 << " to " << output_params.channel_layout() << "; from " 39 << input_params.channels() << " channels to " 40 << output_params.channels() << " channels."; 41 channel_mixer_.reset(new ChannelMixer(input_params, output_params)); 42 43 // Pare off data as early as we can for efficiency. 44 downmix_early_ = input_params.channels() > output_params.channels(); 45 } 46 47 // Only resample if necessary since it's expensive. 48 if (input_params.sample_rate() != output_params.sample_rate()) { 49 DVLOG(1) << "Resampling from " << input_params.sample_rate() << " to " 50 << output_params.sample_rate(); 51 const int request_size = disable_fifo ? SincResampler::kDefaultRequestSize : 52 input_params.frames_per_buffer(); 53 const double io_sample_rate_ratio = 54 input_params.sample_rate() / 55 static_cast<double>(output_params.sample_rate()); 56 resampler_.reset(new MultiChannelResampler( 57 downmix_early_ ? output_params.channels() : input_params.channels(), 58 io_sample_rate_ratio, 59 request_size, 60 base::Bind(&AudioConverter::ProvideInput, base::Unretained(this)))); 61 } 62 63 input_frame_duration_ = base::TimeDelta::FromMicroseconds( 64 base::Time::kMicrosecondsPerSecond / 65 static_cast<double>(input_params.sample_rate())); 66 output_frame_duration_ = base::TimeDelta::FromMicroseconds( 67 base::Time::kMicrosecondsPerSecond / 68 static_cast<double>(output_params.sample_rate())); 69 70 // The resampler can be configured to work with a specific request size, so a 71 // FIFO is not necessary when resampling. 72 if (disable_fifo || resampler_) 73 return; 74 75 // Since the output device may want a different buffer size than the caller 76 // asked for, we need to use a FIFO to ensure that both sides read in chunk 77 // sizes they're configured for. 78 if (input_params.frames_per_buffer() != output_params.frames_per_buffer()) { 79 DVLOG(1) << "Rebuffering from " << input_params.frames_per_buffer() 80 << " to " << output_params.frames_per_buffer(); 81 chunk_size_ = input_params.frames_per_buffer(); 82 audio_fifo_.reset(new AudioPullFifo( 83 downmix_early_ ? output_params.channels() : input_params.channels(), 84 chunk_size_, 85 base::Bind(&AudioConverter::SourceCallback, base::Unretained(this)))); 86 } 87 } 88 89 AudioConverter::~AudioConverter() {} 90 91 void AudioConverter::AddInput(InputCallback* input) { 92 DCHECK(std::find(transform_inputs_.begin(), transform_inputs_.end(), input) == 93 transform_inputs_.end()); 94 transform_inputs_.push_back(input); 95 } 96 97 void AudioConverter::RemoveInput(InputCallback* input) { 98 DCHECK(std::find(transform_inputs_.begin(), transform_inputs_.end(), input) != 99 transform_inputs_.end()); 100 transform_inputs_.remove(input); 101 102 if (transform_inputs_.empty()) 103 Reset(); 104 } 105 106 void AudioConverter::Reset() { 107 if (audio_fifo_) 108 audio_fifo_->Clear(); 109 if (resampler_) 110 resampler_->Flush(); 111 } 112 113 int AudioConverter::ChunkSize() const { 114 if (!resampler_) 115 return chunk_size_; 116 return resampler_->ChunkSize(); 117 } 118 119 void AudioConverter::ConvertWithDelay(const base::TimeDelta& initial_delay, 120 AudioBus* dest) { 121 initial_delay_ = initial_delay; 122 123 if (transform_inputs_.empty()) { 124 dest->Zero(); 125 return; 126 } 127 128 // Determine if channel mixing should be done and if it should be done before 129 // or after resampling. If it's possible to reduce the channel count prior to 130 // resampling we can save a lot of processing time. Vice versa, we don't want 131 // to increase the channel count prior to resampling for the same reason. 132 bool needs_mixing = channel_mixer_ && !downmix_early_; 133 134 if (needs_mixing) 135 CreateUnmixedAudioIfNecessary(dest->frames()); 136 137 AudioBus* temp_dest = needs_mixing ? unmixed_audio_.get() : dest; 138 DCHECK(temp_dest); 139 140 // Figure out which method to call based on whether we're resampling and 141 // rebuffering, just resampling, or just mixing. We want to avoid any extra 142 // steps when possible since we may be converting audio data in real time. 143 if (!resampler_ && !audio_fifo_) { 144 SourceCallback(0, temp_dest); 145 } else { 146 if (resampler_) 147 resampler_->Resample(temp_dest->frames(), temp_dest); 148 else 149 ProvideInput(0, temp_dest); 150 } 151 152 // Finally upmix the channels if we didn't do so earlier. 153 if (needs_mixing) { 154 DCHECK_EQ(temp_dest->frames(), dest->frames()); 155 channel_mixer_->Transform(temp_dest, dest); 156 } 157 } 158 159 void AudioConverter::Convert(AudioBus* dest) { 160 ConvertWithDelay(base::TimeDelta::FromMilliseconds(0), dest); 161 } 162 163 void AudioConverter::SourceCallback(int fifo_frame_delay, AudioBus* dest) { 164 const bool needs_downmix = channel_mixer_ && downmix_early_; 165 166 if (!mixer_input_audio_bus_ || 167 mixer_input_audio_bus_->frames() != dest->frames()) { 168 mixer_input_audio_bus_ = 169 AudioBus::Create(input_channel_count_, dest->frames()); 170 } 171 172 // If we're downmixing early we need a temporary AudioBus which matches 173 // the the input channel count and input frame size since we're passing 174 // |unmixed_audio_| directly to the |source_callback_|. 175 if (needs_downmix) 176 CreateUnmixedAudioIfNecessary(dest->frames()); 177 178 AudioBus* const temp_dest = needs_downmix ? unmixed_audio_.get() : dest; 179 180 // Sanity check our inputs. 181 DCHECK_EQ(temp_dest->frames(), mixer_input_audio_bus_->frames()); 182 DCHECK_EQ(temp_dest->channels(), mixer_input_audio_bus_->channels()); 183 184 // Calculate the buffer delay for this callback. 185 base::TimeDelta buffer_delay = initial_delay_; 186 if (resampler_) { 187 buffer_delay += base::TimeDelta::FromMicroseconds( 188 resampler_frame_delay_ * output_frame_duration_.InMicroseconds()); 189 } 190 if (audio_fifo_) { 191 buffer_delay += base::TimeDelta::FromMicroseconds( 192 fifo_frame_delay * input_frame_duration_.InMicroseconds()); 193 } 194 195 // If we only have a single input, avoid an extra copy. 196 AudioBus* const provide_input_dest = 197 transform_inputs_.size() == 1 ? temp_dest : mixer_input_audio_bus_.get(); 198 199 // Have each mixer render its data into an output buffer then mix the result. 200 for (InputCallbackSet::iterator it = transform_inputs_.begin(); 201 it != transform_inputs_.end(); ++it) { 202 InputCallback* input = *it; 203 204 const float volume = input->ProvideInput(provide_input_dest, buffer_delay); 205 206 // Optimize the most common single input, full volume case. 207 if (it == transform_inputs_.begin()) { 208 if (volume == 1.0f) { 209 if (temp_dest != provide_input_dest) 210 provide_input_dest->CopyTo(temp_dest); 211 } else if (volume > 0) { 212 for (int i = 0; i < provide_input_dest->channels(); ++i) { 213 vector_math::FMUL( 214 provide_input_dest->channel(i), volume, 215 provide_input_dest->frames(), temp_dest->channel(i)); 216 } 217 } else { 218 // Zero |temp_dest| otherwise, so we're mixing into a clean buffer. 219 temp_dest->Zero(); 220 } 221 222 continue; 223 } 224 225 // Volume adjust and mix each mixer input into |temp_dest| after rendering. 226 if (volume > 0) { 227 for (int i = 0; i < mixer_input_audio_bus_->channels(); ++i) { 228 vector_math::FMAC( 229 mixer_input_audio_bus_->channel(i), volume, 230 mixer_input_audio_bus_->frames(), temp_dest->channel(i)); 231 } 232 } 233 } 234 235 if (needs_downmix) { 236 DCHECK_EQ(temp_dest->frames(), dest->frames()); 237 channel_mixer_->Transform(temp_dest, dest); 238 } 239 } 240 241 void AudioConverter::ProvideInput(int resampler_frame_delay, AudioBus* dest) { 242 resampler_frame_delay_ = resampler_frame_delay; 243 if (audio_fifo_) 244 audio_fifo_->Consume(dest, dest->frames()); 245 else 246 SourceCallback(0, dest); 247 } 248 249 void AudioConverter::CreateUnmixedAudioIfNecessary(int frames) { 250 if (!unmixed_audio_ || unmixed_audio_->frames() != frames) 251 unmixed_audio_ = AudioBus::Create(input_channel_count_, frames); 252 } 253 254 } // namespace media 255