1 /* 2 ** 3 ** Copyright 2007, The Android Open Source Project 4 ** 5 ** Licensed under the Apache License, Version 2.0 (the "License"); 6 ** you may not use this file except in compliance with the License. 7 ** You may obtain a copy of the License at 8 ** 9 ** http://www.apache.org/licenses/LICENSE-2.0 10 ** 11 ** Unless required by applicable law or agreed to in writing, software 12 ** distributed under the License is distributed on an "AS IS" BASIS, 13 ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 ** See the License for the specific language governing permissions and 15 ** limitations under the License. 16 */ 17 18 #ifndef ANDROID_AUDIO_MIXER_H 19 #define ANDROID_AUDIO_MIXER_H 20 21 #include <map> 22 #include <pthread.h> 23 #include <sstream> 24 #include <stdint.h> 25 #include <sys/types.h> 26 #include <unordered_map> 27 #include <vector> 28 29 #include <android/os/IExternalVibratorService.h> 30 #include <media/AudioBufferProvider.h> 31 #include <media/AudioResampler.h> 32 #include <media/AudioResamplerPublic.h> 33 #include <media/BufferProviders.h> 34 #include <system/audio.h> 35 #include <utils/Compat.h> 36 #include <utils/threads.h> 37 38 // FIXME This is actually unity gain, which might not be max in future, expressed in U.12 39 #define MAX_GAIN_INT AudioMixer::UNITY_GAIN_INT 40 41 // This must match frameworks/av/services/audioflinger/Configuration.h 42 #define FLOAT_AUX 43 44 namespace android { 45 46 namespace NBLog { 47 class Writer; 48 } // namespace NBLog 49 50 // ---------------------------------------------------------------------------- 51 52 class AudioMixer 53 { 54 public: 55 // Do not change these unless underlying code changes. 56 // This mixer has a hard-coded upper limit of 8 channels for output. 57 static constexpr uint32_t MAX_NUM_CHANNELS = FCC_8; 58 static constexpr uint32_t MAX_NUM_VOLUMES = FCC_2; // stereo volume only 59 // maximum number of channels supported for the content 60 static const uint32_t MAX_NUM_CHANNELS_TO_DOWNMIX = AUDIO_CHANNEL_COUNT_MAX; 61 62 static const uint16_t UNITY_GAIN_INT = 0x1000; 63 static const CONSTEXPR float UNITY_GAIN_FLOAT = 1.0f; 64 65 enum { // names 66 // setParameter targets 67 TRACK = 0x3000, 68 RESAMPLE = 0x3001, 69 RAMP_VOLUME = 0x3002, // ramp to new volume 70 VOLUME = 0x3003, // don't ramp 71 TIMESTRETCH = 0x3004, 72 73 // set Parameter names 74 // for target TRACK 75 CHANNEL_MASK = 0x4000, 76 FORMAT = 0x4001, 77 MAIN_BUFFER = 0x4002, 78 AUX_BUFFER = 0x4003, 79 DOWNMIX_TYPE = 0X4004, 80 MIXER_FORMAT = 0x4005, // AUDIO_FORMAT_PCM_(FLOAT|16_BIT) 81 MIXER_CHANNEL_MASK = 0x4006, // Channel mask for mixer output 82 // for haptic 83 HAPTIC_ENABLED = 0x4007, // Set haptic data from this track should be played or not. 84 HAPTIC_INTENSITY = 0x4008, // Set the intensity to play haptic data. 85 // for target RESAMPLE 86 SAMPLE_RATE = 0x4100, // Configure sample rate conversion on this track name; 87 // parameter 'value' is the new sample rate in Hz. 88 // Only creates a sample rate converter the first time that 89 // the track sample rate is different from the mix sample rate. 90 // If the new sample rate is the same as the mix sample rate, 91 // and a sample rate converter already exists, 92 // then the sample rate converter remains present but is a no-op. 93 RESET = 0x4101, // Reset sample rate converter without changing sample rate. 94 // This clears out the resampler's input buffer. 95 REMOVE = 0x4102, // Remove the sample rate converter on this track name; 96 // the track is restored to the mix sample rate. 97 // for target RAMP_VOLUME and VOLUME (8 channels max) 98 // FIXME use float for these 3 to improve the dynamic range 99 VOLUME0 = 0x4200, 100 VOLUME1 = 0x4201, 101 AUXLEVEL = 0x4210, 102 // for target TIMESTRETCH 103 PLAYBACK_RATE = 0x4300, // Configure timestretch on this track name; 104 // parameter 'value' is a pointer to the new playback rate. 105 }; 106 107 typedef enum { // Haptic intensity, should keep consistent with VibratorService 108 HAPTIC_SCALE_MUTE = os::IExternalVibratorService::SCALE_MUTE, 109 HAPTIC_SCALE_VERY_LOW = os::IExternalVibratorService::SCALE_VERY_LOW, 110 HAPTIC_SCALE_LOW = os::IExternalVibratorService::SCALE_LOW, 111 HAPTIC_SCALE_NONE = os::IExternalVibratorService::SCALE_NONE, 112 HAPTIC_SCALE_HIGH = os::IExternalVibratorService::SCALE_HIGH, 113 HAPTIC_SCALE_VERY_HIGH = os::IExternalVibratorService::SCALE_VERY_HIGH, 114 } haptic_intensity_t; 115 static constexpr float HAPTIC_SCALE_VERY_LOW_RATIO = 2.0f / 3.0f; 116 static constexpr float HAPTIC_SCALE_LOW_RATIO = 3.0f / 4.0f; 117 static const constexpr float HAPTIC_MAX_AMPLITUDE_FLOAT = 1.0f; 118 119 static inline bool isValidHapticIntensity(haptic_intensity_t hapticIntensity) { 120 switch (hapticIntensity) { 121 case HAPTIC_SCALE_MUTE: 122 case HAPTIC_SCALE_VERY_LOW: 123 case HAPTIC_SCALE_LOW: 124 case HAPTIC_SCALE_NONE: 125 case HAPTIC_SCALE_HIGH: 126 case HAPTIC_SCALE_VERY_HIGH: 127 return true; 128 default: 129 return false; 130 } 131 } 132 133 AudioMixer(size_t frameCount, uint32_t sampleRate) 134 : mSampleRate(sampleRate) 135 , mFrameCount(frameCount) { 136 pthread_once(&sOnceControl, &sInitRoutine); 137 } 138 139 // Create a new track in the mixer. 140 // 141 // \param name a unique user-provided integer associated with the track. 142 // If name already exists, the function will abort. 143 // \param channelMask output channel mask. 144 // \param format PCM format 145 // \param sessionId Session id for the track. Tracks with the same 146 // session id will be submixed together. 147 // 148 // \return OK on success. 149 // BAD_VALUE if the format does not satisfy isValidFormat() 150 // or the channelMask does not satisfy isValidChannelMask(). 151 status_t create( 152 int name, audio_channel_mask_t channelMask, audio_format_t format, int sessionId); 153 154 bool exists(int name) const { 155 return mTracks.count(name) > 0; 156 } 157 158 // Free an allocated track by name. 159 void destroy(int name); 160 161 // Enable or disable an allocated track by name 162 void enable(int name); 163 void disable(int name); 164 165 void setParameter(int name, int target, int param, void *value); 166 167 void setBufferProvider(int name, AudioBufferProvider* bufferProvider); 168 169 void process() { 170 for (const auto &pair : mTracks) { 171 // Clear contracted buffer before processing if contracted channels are saved 172 const std::shared_ptr<Track> &t = pair.second; 173 if (t->mKeepContractedChannels) { 174 t->clearContractedBuffer(); 175 } 176 } 177 (this->*mHook)(); 178 processHapticData(); 179 } 180 181 size_t getUnreleasedFrames(int name) const; 182 183 std::string trackNames() const { 184 std::stringstream ss; 185 for (const auto &pair : mTracks) { 186 ss << pair.first << " "; 187 } 188 return ss.str(); 189 } 190 191 void setNBLogWriter(NBLog::Writer *logWriter) { 192 mNBLogWriter = logWriter; 193 } 194 195 static inline bool isValidFormat(audio_format_t format) { 196 switch (format) { 197 case AUDIO_FORMAT_PCM_8_BIT: 198 case AUDIO_FORMAT_PCM_16_BIT: 199 case AUDIO_FORMAT_PCM_24_BIT_PACKED: 200 case AUDIO_FORMAT_PCM_32_BIT: 201 case AUDIO_FORMAT_PCM_FLOAT: 202 return true; 203 default: 204 return false; 205 } 206 } 207 208 static inline bool isValidChannelMask(audio_channel_mask_t channelMask) { 209 return audio_channel_mask_is_valid(channelMask); // the RemixBufferProvider is flexible. 210 } 211 212 private: 213 214 /* For multi-format functions (calls template functions 215 * in AudioMixerOps.h). The template parameters are as follows: 216 * 217 * MIXTYPE (see AudioMixerOps.h MIXTYPE_* enumeration) 218 * USEFLOATVOL (set to true if float volume is used) 219 * ADJUSTVOL (set to true if volume ramp parameters needs adjustment afterwards) 220 * TO: int32_t (Q4.27) or float 221 * TI: int32_t (Q4.27) or int16_t (Q0.15) or float 222 * TA: int32_t (Q4.27) 223 */ 224 225 enum { 226 // FIXME this representation permits up to 8 channels 227 NEEDS_CHANNEL_COUNT__MASK = 0x00000007, 228 }; 229 230 enum { 231 NEEDS_CHANNEL_1 = 0x00000000, // mono 232 NEEDS_CHANNEL_2 = 0x00000001, // stereo 233 234 // sample format is not explicitly specified, and is assumed to be AUDIO_FORMAT_PCM_16_BIT 235 236 NEEDS_MUTE = 0x00000100, 237 NEEDS_RESAMPLE = 0x00001000, 238 NEEDS_AUX = 0x00010000, 239 }; 240 241 // hook types 242 enum { 243 PROCESSTYPE_NORESAMPLEONETRACK, // others set elsewhere 244 }; 245 246 enum { 247 TRACKTYPE_NOP, 248 TRACKTYPE_RESAMPLE, 249 TRACKTYPE_NORESAMPLE, 250 TRACKTYPE_NORESAMPLEMONO, 251 }; 252 253 // process hook functionality 254 using process_hook_t = void(AudioMixer::*)(); 255 256 struct Track; 257 using hook_t = void(Track::*)(int32_t* output, size_t numOutFrames, int32_t* temp, int32_t* aux); 258 259 struct Track { 260 Track() 261 : bufferProvider(nullptr) 262 { 263 // TODO: move additional initialization here. 264 } 265 266 ~Track() 267 { 268 // bufferProvider, mInputBufferProvider need not be deleted. 269 mResampler.reset(nullptr); 270 // Ensure the order of destruction of buffer providers as they 271 // release the upstream provider in the destructor. 272 mTimestretchBufferProvider.reset(nullptr); 273 mPostDownmixReformatBufferProvider.reset(nullptr); 274 mDownmixerBufferProvider.reset(nullptr); 275 mReformatBufferProvider.reset(nullptr); 276 mContractChannelsNonDestructiveBufferProvider.reset(nullptr); 277 mAdjustChannelsBufferProvider.reset(nullptr); 278 } 279 280 bool needsRamp() { return (volumeInc[0] | volumeInc[1] | auxInc) != 0; } 281 bool setResampler(uint32_t trackSampleRate, uint32_t devSampleRate); 282 bool doesResample() const { return mResampler.get() != nullptr; } 283 void resetResampler() { if (mResampler.get() != nullptr) mResampler->reset(); } 284 void adjustVolumeRamp(bool aux, bool useFloat = false); 285 size_t getUnreleasedFrames() const { return mResampler.get() != nullptr ? 286 mResampler->getUnreleasedFrames() : 0; }; 287 288 status_t prepareForDownmix(); 289 void unprepareForDownmix(); 290 status_t prepareForReformat(); 291 void unprepareForReformat(); 292 status_t prepareForAdjustChannels(); 293 void unprepareForAdjustChannels(); 294 status_t prepareForAdjustChannelsNonDestructive(size_t frames); 295 void unprepareForAdjustChannelsNonDestructive(); 296 void clearContractedBuffer(); 297 bool setPlaybackRate(const AudioPlaybackRate &playbackRate); 298 void reconfigureBufferProviders(); 299 300 static hook_t getTrackHook(int trackType, uint32_t channelCount, 301 audio_format_t mixerInFormat, audio_format_t mixerOutFormat); 302 303 void track__nop(int32_t* out, size_t numFrames, int32_t* temp, int32_t* aux); 304 305 template <int MIXTYPE, bool USEFLOATVOL, bool ADJUSTVOL, 306 typename TO, typename TI, typename TA> 307 void volumeMix(TO *out, size_t outFrames, const TI *in, TA *aux, bool ramp); 308 309 uint32_t needs; 310 311 // TODO: Eventually remove legacy integer volume settings 312 union { 313 int16_t volume[MAX_NUM_VOLUMES]; // U4.12 fixed point (top bit should be zero) 314 int32_t volumeRL; 315 }; 316 317 int32_t prevVolume[MAX_NUM_VOLUMES]; 318 int32_t volumeInc[MAX_NUM_VOLUMES]; 319 int32_t auxInc; 320 int32_t prevAuxLevel; 321 int16_t auxLevel; // 0 <= auxLevel <= MAX_GAIN_INT, but signed for mul performance 322 323 uint16_t frameCount; 324 325 uint8_t channelCount; // 1 or 2, redundant with (needs & NEEDS_CHANNEL_COUNT__MASK) 326 uint8_t unused_padding; // formerly format, was always 16 327 uint16_t enabled; // actually bool 328 audio_channel_mask_t channelMask; 329 330 // actual buffer provider used by the track hooks, see DownmixerBufferProvider below 331 // for how the Track buffer provider is wrapped by another one when dowmixing is required 332 AudioBufferProvider* bufferProvider; 333 334 mutable AudioBufferProvider::Buffer buffer; // 8 bytes 335 336 hook_t hook; 337 const void *mIn; // current location in buffer 338 339 std::unique_ptr<AudioResampler> mResampler; 340 uint32_t sampleRate; 341 int32_t* mainBuffer; 342 int32_t* auxBuffer; 343 344 /* Buffer providers are constructed to translate the track input data as needed. 345 * 346 * TODO: perhaps make a single PlaybackConverterProvider class to move 347 * all pre-mixer track buffer conversions outside the AudioMixer class. 348 * 349 * 1) mInputBufferProvider: The AudioTrack buffer provider. 350 * 2) mAdjustChannelsBufferProvider: Expands or contracts sample data from one interleaved 351 * channel format to another. Expanded channels are filled with zeros and put at the end 352 * of each audio frame. Contracted channels are copied to the end of the buffer. 353 * 3) mContractChannelsNonDestructiveBufferProvider: Non-destructively contract sample data. 354 * This is currently using at audio-haptic coupled playback to separate audio and haptic 355 * data. Contracted channels could be written to given buffer. 356 * 4) mReformatBufferProvider: If not NULL, performs the audio reformat to 357 * match either mMixerInFormat or mDownmixRequiresFormat, if the downmixer 358 * requires reformat. For example, it may convert floating point input to 359 * PCM_16_bit if that's required by the downmixer. 360 * 5) mDownmixerBufferProvider: If not NULL, performs the channel remixing to match 361 * the number of channels required by the mixer sink. 362 * 6) mPostDownmixReformatBufferProvider: If not NULL, performs reformatting from 363 * the downmixer requirements to the mixer engine input requirements. 364 * 7) mTimestretchBufferProvider: Adds timestretching for playback rate 365 */ 366 AudioBufferProvider* mInputBufferProvider; // externally provided buffer provider. 367 // TODO: combine mAdjustChannelsBufferProvider and 368 // mContractChannelsNonDestructiveBufferProvider 369 std::unique_ptr<PassthruBufferProvider> mAdjustChannelsBufferProvider; 370 std::unique_ptr<PassthruBufferProvider> mContractChannelsNonDestructiveBufferProvider; 371 std::unique_ptr<PassthruBufferProvider> mReformatBufferProvider; 372 std::unique_ptr<PassthruBufferProvider> mDownmixerBufferProvider; 373 std::unique_ptr<PassthruBufferProvider> mPostDownmixReformatBufferProvider; 374 std::unique_ptr<PassthruBufferProvider> mTimestretchBufferProvider; 375 376 int32_t sessionId; 377 378 audio_format_t mMixerFormat; // output mix format: AUDIO_FORMAT_PCM_(FLOAT|16_BIT) 379 audio_format_t mFormat; // input track format 380 audio_format_t mMixerInFormat; // mix internal format AUDIO_FORMAT_PCM_(FLOAT|16_BIT) 381 // each track must be converted to this format. 382 audio_format_t mDownmixRequiresFormat; // required downmixer format 383 // AUDIO_FORMAT_PCM_16_BIT if 16 bit necessary 384 // AUDIO_FORMAT_INVALID if no required format 385 386 float mVolume[MAX_NUM_VOLUMES]; // floating point set volume 387 float mPrevVolume[MAX_NUM_VOLUMES]; // floating point previous volume 388 float mVolumeInc[MAX_NUM_VOLUMES]; // floating point volume increment 389 390 float mAuxLevel; // floating point set aux level 391 float mPrevAuxLevel; // floating point prev aux level 392 float mAuxInc; // floating point aux increment 393 394 audio_channel_mask_t mMixerChannelMask; 395 uint32_t mMixerChannelCount; 396 397 AudioPlaybackRate mPlaybackRate; 398 399 // Haptic 400 bool mHapticPlaybackEnabled; 401 haptic_intensity_t mHapticIntensity; 402 audio_channel_mask_t mHapticChannelMask; 403 uint32_t mHapticChannelCount; 404 audio_channel_mask_t mMixerHapticChannelMask; 405 uint32_t mMixerHapticChannelCount; 406 uint32_t mAdjustInChannelCount; 407 uint32_t mAdjustOutChannelCount; 408 uint32_t mAdjustNonDestructiveInChannelCount; 409 uint32_t mAdjustNonDestructiveOutChannelCount; 410 bool mKeepContractedChannels; 411 412 float getHapticScaleGamma() const { 413 // Need to keep consistent with the value in VibratorService. 414 switch (mHapticIntensity) { 415 case HAPTIC_SCALE_VERY_LOW: 416 return 2.0f; 417 case HAPTIC_SCALE_LOW: 418 return 1.5f; 419 case HAPTIC_SCALE_HIGH: 420 return 0.5f; 421 case HAPTIC_SCALE_VERY_HIGH: 422 return 0.25f; 423 default: 424 return 1.0f; 425 } 426 } 427 428 float getHapticMaxAmplitudeRatio() const { 429 // Need to keep consistent with the value in VibratorService. 430 switch (mHapticIntensity) { 431 case HAPTIC_SCALE_VERY_LOW: 432 return HAPTIC_SCALE_VERY_LOW_RATIO; 433 case HAPTIC_SCALE_LOW: 434 return HAPTIC_SCALE_LOW_RATIO; 435 case HAPTIC_SCALE_NONE: 436 case HAPTIC_SCALE_HIGH: 437 case HAPTIC_SCALE_VERY_HIGH: 438 return 1.0f; 439 default: 440 return 0.0f; 441 } 442 } 443 444 private: 445 // hooks 446 void track__genericResample(int32_t* out, size_t numFrames, int32_t* temp, int32_t* aux); 447 void track__16BitsStereo(int32_t* out, size_t numFrames, int32_t* temp, int32_t* aux); 448 void track__16BitsMono(int32_t* out, size_t numFrames, int32_t* temp, int32_t* aux); 449 450 void volumeRampStereo(int32_t* out, size_t frameCount, int32_t* temp, int32_t* aux); 451 void volumeStereo(int32_t* out, size_t frameCount, int32_t* temp, int32_t* aux); 452 453 // multi-format track hooks 454 template <int MIXTYPE, typename TO, typename TI, typename TA> 455 void track__Resample(TO* out, size_t frameCount, TO* temp __unused, TA* aux); 456 template <int MIXTYPE, typename TO, typename TI, typename TA> 457 void track__NoResample(TO* out, size_t frameCount, TO* temp __unused, TA* aux); 458 }; 459 460 // TODO: remove BLOCKSIZE unit of processing - it isn't needed anymore. 461 static constexpr int BLOCKSIZE = 16; 462 463 bool setChannelMasks(int name, 464 audio_channel_mask_t trackChannelMask, audio_channel_mask_t mixerChannelMask); 465 466 // Called when track info changes and a new process hook should be determined. 467 void invalidate() { 468 mHook = &AudioMixer::process__validate; 469 } 470 471 void process__validate(); 472 void process__nop(); 473 void process__genericNoResampling(); 474 void process__genericResampling(); 475 void process__oneTrack16BitsStereoNoResampling(); 476 477 template <int MIXTYPE, typename TO, typename TI, typename TA> 478 void process__noResampleOneTrack(); 479 480 void processHapticData(); 481 482 static process_hook_t getProcessHook(int processType, uint32_t channelCount, 483 audio_format_t mixerInFormat, audio_format_t mixerOutFormat); 484 485 static void convertMixerFormat(void *out, audio_format_t mixerOutFormat, 486 void *in, audio_format_t mixerInFormat, size_t sampleCount); 487 488 static void sInitRoutine(); 489 490 // initialization constants 491 const uint32_t mSampleRate; 492 const size_t mFrameCount; 493 494 NBLog::Writer *mNBLogWriter = nullptr; // associated NBLog::Writer 495 496 process_hook_t mHook = &AudioMixer::process__nop; // one of process__*, never nullptr 497 498 // the size of the type (int32_t) should be the largest of all types supported 499 // by the mixer. 500 std::unique_ptr<int32_t[]> mOutputTemp; 501 std::unique_ptr<int32_t[]> mResampleTemp; 502 503 // track names grouped by main buffer, in no particular order of main buffer. 504 // however names for a particular main buffer are in order (by construction). 505 std::unordered_map<void * /* mainBuffer */, std::vector<int /* name */>> mGroups; 506 507 // track names that are enabled, in increasing order (by construction). 508 std::vector<int /* name */> mEnabled; 509 510 // track smart pointers, by name, in increasing order of name. 511 std::map<int /* name */, std::shared_ptr<Track>> mTracks; 512 513 static pthread_once_t sOnceControl; // initialized in constructor by first new 514 }; 515 516 // ---------------------------------------------------------------------------- 517 } // namespace android 518 519 #endif // ANDROID_AUDIO_MIXER_H 520
