1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "webrtc/modules/audio_conference_mixer/include/audio_conference_mixer_defines.h" 12 #include "webrtc/modules/audio_conference_mixer/source/audio_conference_mixer_impl.h" 13 #include "webrtc/modules/audio_conference_mixer/source/audio_frame_manipulator.h" 14 #include "webrtc/modules/audio_processing/include/audio_processing.h" 15 #include "webrtc/modules/utility/include/audio_frame_operations.h" 16 #include "webrtc/system_wrappers/include/critical_section_wrapper.h" 17 #include "webrtc/system_wrappers/include/trace.h" 18 19 namespace webrtc { 20 namespace { 21 22 struct ParticipantFramePair { 23 MixerParticipant* participant; 24 AudioFrame* audioFrame; 25 }; 26 27 typedef std::list<ParticipantFramePair*> ParticipantFramePairList; 28 29 // Mix |frame| into |mixed_frame|, with saturation protection and upmixing. 30 // These effects are applied to |frame| itself prior to mixing. Assumes that 31 // |mixed_frame| always has at least as many channels as |frame|. Supports 32 // stereo at most. 33 // 34 // TODO(andrew): consider not modifying |frame| here. 35 void MixFrames(AudioFrame* mixed_frame, AudioFrame* frame, bool use_limiter) { 36 assert(mixed_frame->num_channels_ >= frame->num_channels_); 37 if (use_limiter) { 38 // Divide by two to avoid saturation in the mixing. 39 // This is only meaningful if the limiter will be used. 40 *frame >>= 1; 41 } 42 if (mixed_frame->num_channels_ > frame->num_channels_) { 43 // We only support mono-to-stereo. 44 assert(mixed_frame->num_channels_ == 2 && 45 frame->num_channels_ == 1); 46 AudioFrameOperations::MonoToStereo(frame); 47 } 48 49 *mixed_frame += *frame; 50 } 51 52 // Return the max number of channels from a |list| composed of AudioFrames. 53 size_t MaxNumChannels(const AudioFrameList* list) { 54 size_t max_num_channels = 1; 55 for (AudioFrameList::const_iterator iter = list->begin(); 56 iter != list->end(); 57 ++iter) { 58 max_num_channels = std::max(max_num_channels, (*iter)->num_channels_); 59 } 60 return max_num_channels; 61 } 62 63 } // namespace 64 65 MixerParticipant::MixerParticipant() 66 : _mixHistory(new MixHistory()) { 67 } 68 69 MixerParticipant::~MixerParticipant() { 70 delete _mixHistory; 71 } 72 73 bool MixerParticipant::IsMixed() const { 74 return _mixHistory->IsMixed(); 75 } 76 77 MixHistory::MixHistory() 78 : _isMixed(0) { 79 } 80 81 MixHistory::~MixHistory() { 82 } 83 84 bool MixHistory::IsMixed() const { 85 return _isMixed; 86 } 87 88 bool MixHistory::WasMixed() const { 89 // Was mixed is the same as is mixed depending on perspective. This function 90 // is for the perspective of AudioConferenceMixerImpl. 91 return IsMixed(); 92 } 93 94 int32_t MixHistory::SetIsMixed(const bool mixed) { 95 _isMixed = mixed; 96 return 0; 97 } 98 99 void MixHistory::ResetMixedStatus() { 100 _isMixed = false; 101 } 102 103 AudioConferenceMixer* AudioConferenceMixer::Create(int id) { 104 AudioConferenceMixerImpl* mixer = new AudioConferenceMixerImpl(id); 105 if(!mixer->Init()) { 106 delete mixer; 107 return NULL; 108 } 109 return mixer; 110 } 111 112 AudioConferenceMixerImpl::AudioConferenceMixerImpl(int id) 113 : _id(id), 114 _minimumMixingFreq(kLowestPossible), 115 _mixReceiver(NULL), 116 _outputFrequency(kDefaultFrequency), 117 _sampleSize(0), 118 _audioFramePool(NULL), 119 _participantList(), 120 _additionalParticipantList(), 121 _numMixedParticipants(0), 122 use_limiter_(true), 123 _timeStamp(0), 124 _timeScheduler(kProcessPeriodicityInMs), 125 _processCalls(0) {} 126 127 bool AudioConferenceMixerImpl::Init() { 128 _crit.reset(CriticalSectionWrapper::CreateCriticalSection()); 129 if (_crit.get() == NULL) 130 return false; 131 132 _cbCrit.reset(CriticalSectionWrapper::CreateCriticalSection()); 133 if(_cbCrit.get() == NULL) 134 return false; 135 136 Config config; 137 config.Set<ExperimentalAgc>(new ExperimentalAgc(false)); 138 _limiter.reset(AudioProcessing::Create(config)); 139 if(!_limiter.get()) 140 return false; 141 142 MemoryPool<AudioFrame>::CreateMemoryPool(_audioFramePool, 143 DEFAULT_AUDIO_FRAME_POOLSIZE); 144 if(_audioFramePool == NULL) 145 return false; 146 147 if(SetOutputFrequency(kDefaultFrequency) == -1) 148 return false; 149 150 if(_limiter->gain_control()->set_mode(GainControl::kFixedDigital) != 151 _limiter->kNoError) 152 return false; 153 154 // We smoothly limit the mixed frame to -7 dbFS. -6 would correspond to the 155 // divide-by-2 but -7 is used instead to give a bit of headroom since the 156 // AGC is not a hard limiter. 157 if(_limiter->gain_control()->set_target_level_dbfs(7) != _limiter->kNoError) 158 return false; 159 160 if(_limiter->gain_control()->set_compression_gain_db(0) 161 != _limiter->kNoError) 162 return false; 163 164 if(_limiter->gain_control()->enable_limiter(true) != _limiter->kNoError) 165 return false; 166 167 if(_limiter->gain_control()->Enable(true) != _limiter->kNoError) 168 return false; 169 170 return true; 171 } 172 173 AudioConferenceMixerImpl::~AudioConferenceMixerImpl() { 174 MemoryPool<AudioFrame>::DeleteMemoryPool(_audioFramePool); 175 assert(_audioFramePool == NULL); 176 } 177 178 // Process should be called every kProcessPeriodicityInMs ms 179 int64_t AudioConferenceMixerImpl::TimeUntilNextProcess() { 180 int64_t timeUntilNextProcess = 0; 181 CriticalSectionScoped cs(_crit.get()); 182 if(_timeScheduler.TimeToNextUpdate(timeUntilNextProcess) != 0) { 183 WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id, 184 "failed in TimeToNextUpdate() call"); 185 // Sanity check 186 assert(false); 187 return -1; 188 } 189 return timeUntilNextProcess; 190 } 191 192 int32_t AudioConferenceMixerImpl::Process() { 193 size_t remainingParticipantsAllowedToMix = 194 kMaximumAmountOfMixedParticipants; 195 { 196 CriticalSectionScoped cs(_crit.get()); 197 assert(_processCalls == 0); 198 _processCalls++; 199 200 // Let the scheduler know that we are running one iteration. 201 _timeScheduler.UpdateScheduler(); 202 } 203 204 AudioFrameList mixList; 205 AudioFrameList rampOutList; 206 AudioFrameList additionalFramesList; 207 std::map<int, MixerParticipant*> mixedParticipantsMap; 208 { 209 CriticalSectionScoped cs(_cbCrit.get()); 210 211 int32_t lowFreq = GetLowestMixingFrequency(); 212 // SILK can run in 12 kHz and 24 kHz. These frequencies are not 213 // supported so use the closest higher frequency to not lose any 214 // information. 215 // TODO(henrike): this is probably more appropriate to do in 216 // GetLowestMixingFrequency(). 217 if (lowFreq == 12000) { 218 lowFreq = 16000; 219 } else if (lowFreq == 24000) { 220 lowFreq = 32000; 221 } 222 if(lowFreq <= 0) { 223 CriticalSectionScoped cs(_crit.get()); 224 _processCalls--; 225 return 0; 226 } else { 227 switch(lowFreq) { 228 case 8000: 229 if(OutputFrequency() != kNbInHz) { 230 SetOutputFrequency(kNbInHz); 231 } 232 break; 233 case 16000: 234 if(OutputFrequency() != kWbInHz) { 235 SetOutputFrequency(kWbInHz); 236 } 237 break; 238 case 32000: 239 if(OutputFrequency() != kSwbInHz) { 240 SetOutputFrequency(kSwbInHz); 241 } 242 break; 243 case 48000: 244 if(OutputFrequency() != kFbInHz) { 245 SetOutputFrequency(kFbInHz); 246 } 247 break; 248 default: 249 assert(false); 250 251 CriticalSectionScoped cs(_crit.get()); 252 _processCalls--; 253 return -1; 254 } 255 } 256 257 UpdateToMix(&mixList, &rampOutList, &mixedParticipantsMap, 258 &remainingParticipantsAllowedToMix); 259 260 GetAdditionalAudio(&additionalFramesList); 261 UpdateMixedStatus(mixedParticipantsMap); 262 } 263 264 // Get an AudioFrame for mixing from the memory pool. 265 AudioFrame* mixedAudio = NULL; 266 if(_audioFramePool->PopMemory(mixedAudio) == -1) { 267 WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id, 268 "failed PopMemory() call"); 269 assert(false); 270 return -1; 271 } 272 273 int retval = 0; 274 { 275 CriticalSectionScoped cs(_crit.get()); 276 277 // TODO(henrike): it might be better to decide the number of channels 278 // with an API instead of dynamically. 279 280 // Find the max channels over all mixing lists. 281 const size_t num_mixed_channels = std::max(MaxNumChannels(&mixList), 282 std::max(MaxNumChannels(&additionalFramesList), 283 MaxNumChannels(&rampOutList))); 284 285 mixedAudio->UpdateFrame(-1, _timeStamp, NULL, 0, _outputFrequency, 286 AudioFrame::kNormalSpeech, 287 AudioFrame::kVadPassive, num_mixed_channels); 288 289 _timeStamp += static_cast<uint32_t>(_sampleSize); 290 291 // We only use the limiter if it supports the output sample rate and 292 // we're actually mixing multiple streams. 293 use_limiter_ = 294 _numMixedParticipants > 1 && 295 _outputFrequency <= AudioProcessing::kMaxNativeSampleRateHz; 296 297 MixFromList(mixedAudio, mixList); 298 MixAnonomouslyFromList(mixedAudio, additionalFramesList); 299 MixAnonomouslyFromList(mixedAudio, rampOutList); 300 301 if(mixedAudio->samples_per_channel_ == 0) { 302 // Nothing was mixed, set the audio samples to silence. 303 mixedAudio->samples_per_channel_ = _sampleSize; 304 mixedAudio->Mute(); 305 } else { 306 // Only call the limiter if we have something to mix. 307 if(!LimitMixedAudio(mixedAudio)) 308 retval = -1; 309 } 310 } 311 312 { 313 CriticalSectionScoped cs(_cbCrit.get()); 314 if(_mixReceiver != NULL) { 315 const AudioFrame** dummy = NULL; 316 _mixReceiver->NewMixedAudio( 317 _id, 318 *mixedAudio, 319 dummy, 320 0); 321 } 322 } 323 324 // Reclaim all outstanding memory. 325 _audioFramePool->PushMemory(mixedAudio); 326 ClearAudioFrameList(&mixList); 327 ClearAudioFrameList(&rampOutList); 328 ClearAudioFrameList(&additionalFramesList); 329 { 330 CriticalSectionScoped cs(_crit.get()); 331 _processCalls--; 332 } 333 return retval; 334 } 335 336 int32_t AudioConferenceMixerImpl::RegisterMixedStreamCallback( 337 AudioMixerOutputReceiver* mixReceiver) { 338 CriticalSectionScoped cs(_cbCrit.get()); 339 if(_mixReceiver != NULL) { 340 return -1; 341 } 342 _mixReceiver = mixReceiver; 343 return 0; 344 } 345 346 int32_t AudioConferenceMixerImpl::UnRegisterMixedStreamCallback() { 347 CriticalSectionScoped cs(_cbCrit.get()); 348 if(_mixReceiver == NULL) { 349 return -1; 350 } 351 _mixReceiver = NULL; 352 return 0; 353 } 354 355 int32_t AudioConferenceMixerImpl::SetOutputFrequency( 356 const Frequency& frequency) { 357 CriticalSectionScoped cs(_crit.get()); 358 359 _outputFrequency = frequency; 360 _sampleSize = 361 static_cast<size_t>((_outputFrequency*kProcessPeriodicityInMs) / 1000); 362 363 return 0; 364 } 365 366 AudioConferenceMixer::Frequency 367 AudioConferenceMixerImpl::OutputFrequency() const { 368 CriticalSectionScoped cs(_crit.get()); 369 return _outputFrequency; 370 } 371 372 int32_t AudioConferenceMixerImpl::SetMixabilityStatus( 373 MixerParticipant* participant, bool mixable) { 374 if (!mixable) { 375 // Anonymous participants are in a separate list. Make sure that the 376 // participant is in the _participantList if it is being mixed. 377 SetAnonymousMixabilityStatus(participant, false); 378 } 379 size_t numMixedParticipants; 380 { 381 CriticalSectionScoped cs(_cbCrit.get()); 382 const bool isMixed = 383 IsParticipantInList(*participant, _participantList); 384 // API must be called with a new state. 385 if(!(mixable ^ isMixed)) { 386 WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id, 387 "Mixable is aready %s", 388 isMixed ? "ON" : "off"); 389 return -1; 390 } 391 bool success = false; 392 if(mixable) { 393 success = AddParticipantToList(participant, &_participantList); 394 } else { 395 success = RemoveParticipantFromList(participant, &_participantList); 396 } 397 if(!success) { 398 WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id, 399 "failed to %s participant", 400 mixable ? "add" : "remove"); 401 assert(false); 402 return -1; 403 } 404 405 size_t numMixedNonAnonymous = _participantList.size(); 406 if (numMixedNonAnonymous > kMaximumAmountOfMixedParticipants) { 407 numMixedNonAnonymous = kMaximumAmountOfMixedParticipants; 408 } 409 numMixedParticipants = 410 numMixedNonAnonymous + _additionalParticipantList.size(); 411 } 412 // A MixerParticipant was added or removed. Make sure the scratch 413 // buffer is updated if necessary. 414 // Note: The scratch buffer may only be updated in Process(). 415 CriticalSectionScoped cs(_crit.get()); 416 _numMixedParticipants = numMixedParticipants; 417 return 0; 418 } 419 420 bool AudioConferenceMixerImpl::MixabilityStatus( 421 const MixerParticipant& participant) const { 422 CriticalSectionScoped cs(_cbCrit.get()); 423 return IsParticipantInList(participant, _participantList); 424 } 425 426 int32_t AudioConferenceMixerImpl::SetAnonymousMixabilityStatus( 427 MixerParticipant* participant, bool anonymous) { 428 CriticalSectionScoped cs(_cbCrit.get()); 429 if(IsParticipantInList(*participant, _additionalParticipantList)) { 430 if(anonymous) { 431 return 0; 432 } 433 if(!RemoveParticipantFromList(participant, 434 &_additionalParticipantList)) { 435 WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id, 436 "unable to remove participant from anonymous list"); 437 assert(false); 438 return -1; 439 } 440 return AddParticipantToList(participant, &_participantList) ? 0 : -1; 441 } 442 if(!anonymous) { 443 return 0; 444 } 445 const bool mixable = RemoveParticipantFromList(participant, 446 &_participantList); 447 if(!mixable) { 448 WEBRTC_TRACE( 449 kTraceWarning, 450 kTraceAudioMixerServer, 451 _id, 452 "participant must be registered before turning it into anonymous"); 453 // Setting anonymous status is only possible if MixerParticipant is 454 // already registered. 455 return -1; 456 } 457 return AddParticipantToList(participant, &_additionalParticipantList) ? 458 0 : -1; 459 } 460 461 bool AudioConferenceMixerImpl::AnonymousMixabilityStatus( 462 const MixerParticipant& participant) const { 463 CriticalSectionScoped cs(_cbCrit.get()); 464 return IsParticipantInList(participant, _additionalParticipantList); 465 } 466 467 int32_t AudioConferenceMixerImpl::SetMinimumMixingFrequency( 468 Frequency freq) { 469 // Make sure that only allowed sampling frequencies are used. Use closest 470 // higher sampling frequency to avoid losing information. 471 if (static_cast<int>(freq) == 12000) { 472 freq = kWbInHz; 473 } else if (static_cast<int>(freq) == 24000) { 474 freq = kSwbInHz; 475 } 476 477 if((freq == kNbInHz) || (freq == kWbInHz) || (freq == kSwbInHz) || 478 (freq == kLowestPossible)) { 479 _minimumMixingFreq=freq; 480 return 0; 481 } else { 482 WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id, 483 "SetMinimumMixingFrequency incorrect frequency: %i",freq); 484 assert(false); 485 return -1; 486 } 487 } 488 489 // Check all AudioFrames that are to be mixed. The highest sampling frequency 490 // found is the lowest that can be used without losing information. 491 int32_t AudioConferenceMixerImpl::GetLowestMixingFrequency() const { 492 const int participantListFrequency = 493 GetLowestMixingFrequencyFromList(_participantList); 494 const int anonymousListFrequency = 495 GetLowestMixingFrequencyFromList(_additionalParticipantList); 496 const int highestFreq = 497 (participantListFrequency > anonymousListFrequency) ? 498 participantListFrequency : anonymousListFrequency; 499 // Check if the user specified a lowest mixing frequency. 500 if(_minimumMixingFreq != kLowestPossible) { 501 if(_minimumMixingFreq > highestFreq) { 502 return _minimumMixingFreq; 503 } 504 } 505 return highestFreq; 506 } 507 508 int32_t AudioConferenceMixerImpl::GetLowestMixingFrequencyFromList( 509 const MixerParticipantList& mixList) const { 510 int32_t highestFreq = 8000; 511 for (MixerParticipantList::const_iterator iter = mixList.begin(); 512 iter != mixList.end(); 513 ++iter) { 514 const int32_t neededFrequency = (*iter)->NeededFrequency(_id); 515 if(neededFrequency > highestFreq) { 516 highestFreq = neededFrequency; 517 } 518 } 519 return highestFreq; 520 } 521 522 void AudioConferenceMixerImpl::UpdateToMix( 523 AudioFrameList* mixList, 524 AudioFrameList* rampOutList, 525 std::map<int, MixerParticipant*>* mixParticipantList, 526 size_t* maxAudioFrameCounter) const { 527 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, 528 "UpdateToMix(mixList,rampOutList,mixParticipantList,%d)", 529 *maxAudioFrameCounter); 530 const size_t mixListStartSize = mixList->size(); 531 AudioFrameList activeList; 532 // Struct needed by the passive lists to keep track of which AudioFrame 533 // belongs to which MixerParticipant. 534 ParticipantFramePairList passiveWasNotMixedList; 535 ParticipantFramePairList passiveWasMixedList; 536 for (MixerParticipantList::const_iterator participant = 537 _participantList.begin(); participant != _participantList.end(); 538 ++participant) { 539 // Stop keeping track of passive participants if there are already 540 // enough participants available (they wont be mixed anyway). 541 bool mustAddToPassiveList = (*maxAudioFrameCounter > 542 (activeList.size() + 543 passiveWasMixedList.size() + 544 passiveWasNotMixedList.size())); 545 546 bool wasMixed = false; 547 wasMixed = (*participant)->_mixHistory->WasMixed(); 548 AudioFrame* audioFrame = NULL; 549 if(_audioFramePool->PopMemory(audioFrame) == -1) { 550 WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id, 551 "failed PopMemory() call"); 552 assert(false); 553 return; 554 } 555 audioFrame->sample_rate_hz_ = _outputFrequency; 556 557 if((*participant)->GetAudioFrame(_id, audioFrame) != 0) { 558 WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id, 559 "failed to GetAudioFrame() from participant"); 560 _audioFramePool->PushMemory(audioFrame); 561 continue; 562 } 563 if (_participantList.size() != 1) { 564 // TODO(wu): Issue 3390, add support for multiple participants case. 565 audioFrame->ntp_time_ms_ = -1; 566 } 567 568 // TODO(henrike): this assert triggers in some test cases where SRTP is 569 // used which prevents NetEQ from making a VAD. Temporarily disable this 570 // assert until the problem is fixed on a higher level. 571 // assert(audioFrame->vad_activity_ != AudioFrame::kVadUnknown); 572 if (audioFrame->vad_activity_ == AudioFrame::kVadUnknown) { 573 WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id, 574 "invalid VAD state from participant"); 575 } 576 577 if(audioFrame->vad_activity_ == AudioFrame::kVadActive) { 578 if(!wasMixed) { 579 RampIn(*audioFrame); 580 } 581 582 if(activeList.size() >= *maxAudioFrameCounter) { 583 // There are already more active participants than should be 584 // mixed. Only keep the ones with the highest energy. 585 AudioFrameList::iterator replaceItem; 586 CalculateEnergy(*audioFrame); 587 uint32_t lowestEnergy = audioFrame->energy_; 588 589 bool found_replace_item = false; 590 for (AudioFrameList::iterator iter = activeList.begin(); 591 iter != activeList.end(); 592 ++iter) { 593 CalculateEnergy(**iter); 594 if((*iter)->energy_ < lowestEnergy) { 595 replaceItem = iter; 596 lowestEnergy = (*iter)->energy_; 597 found_replace_item = true; 598 } 599 } 600 if(found_replace_item) { 601 AudioFrame* replaceFrame = *replaceItem; 602 603 bool replaceWasMixed = false; 604 std::map<int, MixerParticipant*>::const_iterator it = 605 mixParticipantList->find(replaceFrame->id_); 606 607 // When a frame is pushed to |activeList| it is also pushed 608 // to mixParticipantList with the frame's id. This means 609 // that the Find call above should never fail. 610 assert(it != mixParticipantList->end()); 611 replaceWasMixed = it->second->_mixHistory->WasMixed(); 612 613 mixParticipantList->erase(replaceFrame->id_); 614 activeList.erase(replaceItem); 615 616 activeList.push_front(audioFrame); 617 (*mixParticipantList)[audioFrame->id_] = *participant; 618 assert(mixParticipantList->size() <= 619 kMaximumAmountOfMixedParticipants); 620 621 if (replaceWasMixed) { 622 RampOut(*replaceFrame); 623 rampOutList->push_back(replaceFrame); 624 assert(rampOutList->size() <= 625 kMaximumAmountOfMixedParticipants); 626 } else { 627 _audioFramePool->PushMemory(replaceFrame); 628 } 629 } else { 630 if(wasMixed) { 631 RampOut(*audioFrame); 632 rampOutList->push_back(audioFrame); 633 assert(rampOutList->size() <= 634 kMaximumAmountOfMixedParticipants); 635 } else { 636 _audioFramePool->PushMemory(audioFrame); 637 } 638 } 639 } else { 640 activeList.push_front(audioFrame); 641 (*mixParticipantList)[audioFrame->id_] = *participant; 642 assert(mixParticipantList->size() <= 643 kMaximumAmountOfMixedParticipants); 644 } 645 } else { 646 if(wasMixed) { 647 ParticipantFramePair* pair = new ParticipantFramePair; 648 pair->audioFrame = audioFrame; 649 pair->participant = *participant; 650 passiveWasMixedList.push_back(pair); 651 } else if(mustAddToPassiveList) { 652 RampIn(*audioFrame); 653 ParticipantFramePair* pair = new ParticipantFramePair; 654 pair->audioFrame = audioFrame; 655 pair->participant = *participant; 656 passiveWasNotMixedList.push_back(pair); 657 } else { 658 _audioFramePool->PushMemory(audioFrame); 659 } 660 } 661 } 662 assert(activeList.size() <= *maxAudioFrameCounter); 663 // At this point it is known which participants should be mixed. Transfer 664 // this information to this functions output parameters. 665 for (AudioFrameList::const_iterator iter = activeList.begin(); 666 iter != activeList.end(); 667 ++iter) { 668 mixList->push_back(*iter); 669 } 670 activeList.clear(); 671 // Always mix a constant number of AudioFrames. If there aren't enough 672 // active participants mix passive ones. Starting with those that was mixed 673 // last iteration. 674 for (ParticipantFramePairList::const_iterator 675 iter = passiveWasMixedList.begin(); iter != passiveWasMixedList.end(); 676 ++iter) { 677 if(mixList->size() < *maxAudioFrameCounter + mixListStartSize) { 678 mixList->push_back((*iter)->audioFrame); 679 (*mixParticipantList)[(*iter)->audioFrame->id_] = 680 (*iter)->participant; 681 assert(mixParticipantList->size() <= 682 kMaximumAmountOfMixedParticipants); 683 } else { 684 _audioFramePool->PushMemory((*iter)->audioFrame); 685 } 686 delete *iter; 687 } 688 // And finally the ones that have not been mixed for a while. 689 for (ParticipantFramePairList::const_iterator iter = 690 passiveWasNotMixedList.begin(); 691 iter != passiveWasNotMixedList.end(); 692 ++iter) { 693 if(mixList->size() < *maxAudioFrameCounter + mixListStartSize) { 694 mixList->push_back((*iter)->audioFrame); 695 (*mixParticipantList)[(*iter)->audioFrame->id_] = 696 (*iter)->participant; 697 assert(mixParticipantList->size() <= 698 kMaximumAmountOfMixedParticipants); 699 } else { 700 _audioFramePool->PushMemory((*iter)->audioFrame); 701 } 702 delete *iter; 703 } 704 assert(*maxAudioFrameCounter + mixListStartSize >= mixList->size()); 705 *maxAudioFrameCounter += mixListStartSize - mixList->size(); 706 } 707 708 void AudioConferenceMixerImpl::GetAdditionalAudio( 709 AudioFrameList* additionalFramesList) const { 710 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, 711 "GetAdditionalAudio(additionalFramesList)"); 712 // The GetAudioFrame() callback may result in the participant being removed 713 // from additionalParticipantList_. If that happens it will invalidate any 714 // iterators. Create a copy of the participants list such that the list of 715 // participants can be traversed safely. 716 MixerParticipantList additionalParticipantList; 717 additionalParticipantList.insert(additionalParticipantList.begin(), 718 _additionalParticipantList.begin(), 719 _additionalParticipantList.end()); 720 721 for (MixerParticipantList::const_iterator participant = 722 additionalParticipantList.begin(); 723 participant != additionalParticipantList.end(); 724 ++participant) { 725 AudioFrame* audioFrame = NULL; 726 if(_audioFramePool->PopMemory(audioFrame) == -1) { 727 WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id, 728 "failed PopMemory() call"); 729 assert(false); 730 return; 731 } 732 audioFrame->sample_rate_hz_ = _outputFrequency; 733 if((*participant)->GetAudioFrame(_id, audioFrame) != 0) { 734 WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id, 735 "failed to GetAudioFrame() from participant"); 736 _audioFramePool->PushMemory(audioFrame); 737 continue; 738 } 739 if(audioFrame->samples_per_channel_ == 0) { 740 // Empty frame. Don't use it. 741 _audioFramePool->PushMemory(audioFrame); 742 continue; 743 } 744 additionalFramesList->push_back(audioFrame); 745 } 746 } 747 748 void AudioConferenceMixerImpl::UpdateMixedStatus( 749 const std::map<int, MixerParticipant*>& mixedParticipantsMap) const { 750 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, 751 "UpdateMixedStatus(mixedParticipantsMap)"); 752 assert(mixedParticipantsMap.size() <= kMaximumAmountOfMixedParticipants); 753 754 // Loop through all participants. If they are in the mix map they 755 // were mixed. 756 for (MixerParticipantList::const_iterator 757 participant =_participantList.begin(); 758 participant != _participantList.end(); 759 ++participant) { 760 bool isMixed = false; 761 for (std::map<int, MixerParticipant*>::const_iterator it = 762 mixedParticipantsMap.begin(); 763 it != mixedParticipantsMap.end(); 764 ++it) { 765 if (it->second == *participant) { 766 isMixed = true; 767 break; 768 } 769 } 770 (*participant)->_mixHistory->SetIsMixed(isMixed); 771 } 772 } 773 774 void AudioConferenceMixerImpl::ClearAudioFrameList( 775 AudioFrameList* audioFrameList) const { 776 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, 777 "ClearAudioFrameList(audioFrameList)"); 778 for (AudioFrameList::iterator iter = audioFrameList->begin(); 779 iter != audioFrameList->end(); 780 ++iter) { 781 _audioFramePool->PushMemory(*iter); 782 } 783 audioFrameList->clear(); 784 } 785 786 void AudioConferenceMixerImpl::UpdateVADPositiveParticipants( 787 AudioFrameList* mixList) const { 788 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, 789 "UpdateVADPositiveParticipants(mixList)"); 790 791 for (AudioFrameList::const_iterator iter = mixList->begin(); 792 iter != mixList->end(); 793 ++iter) { 794 CalculateEnergy(**iter); 795 } 796 } 797 798 bool AudioConferenceMixerImpl::IsParticipantInList( 799 const MixerParticipant& participant, 800 const MixerParticipantList& participantList) const { 801 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, 802 "IsParticipantInList(participant,participantList)"); 803 for (MixerParticipantList::const_iterator iter = participantList.begin(); 804 iter != participantList.end(); 805 ++iter) { 806 if(&participant == *iter) { 807 return true; 808 } 809 } 810 return false; 811 } 812 813 bool AudioConferenceMixerImpl::AddParticipantToList( 814 MixerParticipant* participant, 815 MixerParticipantList* participantList) const { 816 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, 817 "AddParticipantToList(participant, participantList)"); 818 participantList->push_back(participant); 819 // Make sure that the mixed status is correct for new MixerParticipant. 820 participant->_mixHistory->ResetMixedStatus(); 821 return true; 822 } 823 824 bool AudioConferenceMixerImpl::RemoveParticipantFromList( 825 MixerParticipant* participant, 826 MixerParticipantList* participantList) const { 827 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, 828 "RemoveParticipantFromList(participant, participantList)"); 829 for (MixerParticipantList::iterator iter = participantList->begin(); 830 iter != participantList->end(); 831 ++iter) { 832 if(*iter == participant) { 833 participantList->erase(iter); 834 // Participant is no longer mixed, reset to default. 835 participant->_mixHistory->ResetMixedStatus(); 836 return true; 837 } 838 } 839 return false; 840 } 841 842 int32_t AudioConferenceMixerImpl::MixFromList( 843 AudioFrame* mixedAudio, 844 const AudioFrameList& audioFrameList) const { 845 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, 846 "MixFromList(mixedAudio, audioFrameList)"); 847 if(audioFrameList.empty()) return 0; 848 849 uint32_t position = 0; 850 851 if (_numMixedParticipants == 1) { 852 mixedAudio->timestamp_ = audioFrameList.front()->timestamp_; 853 mixedAudio->elapsed_time_ms_ = audioFrameList.front()->elapsed_time_ms_; 854 } else { 855 // TODO(wu): Issue 3390. 856 // Audio frame timestamp is only supported in one channel case. 857 mixedAudio->timestamp_ = 0; 858 mixedAudio->elapsed_time_ms_ = -1; 859 } 860 861 for (AudioFrameList::const_iterator iter = audioFrameList.begin(); 862 iter != audioFrameList.end(); 863 ++iter) { 864 if(position >= kMaximumAmountOfMixedParticipants) { 865 WEBRTC_TRACE( 866 kTraceMemory, 867 kTraceAudioMixerServer, 868 _id, 869 "Trying to mix more than max amount of mixed participants:%d!", 870 kMaximumAmountOfMixedParticipants); 871 // Assert and avoid crash 872 assert(false); 873 position = 0; 874 } 875 MixFrames(mixedAudio, (*iter), use_limiter_); 876 877 position++; 878 } 879 880 return 0; 881 } 882 883 // TODO(andrew): consolidate this function with MixFromList. 884 int32_t AudioConferenceMixerImpl::MixAnonomouslyFromList( 885 AudioFrame* mixedAudio, 886 const AudioFrameList& audioFrameList) const { 887 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, 888 "MixAnonomouslyFromList(mixedAudio, audioFrameList)"); 889 890 if(audioFrameList.empty()) return 0; 891 892 for (AudioFrameList::const_iterator iter = audioFrameList.begin(); 893 iter != audioFrameList.end(); 894 ++iter) { 895 MixFrames(mixedAudio, *iter, use_limiter_); 896 } 897 return 0; 898 } 899 900 bool AudioConferenceMixerImpl::LimitMixedAudio(AudioFrame* mixedAudio) const { 901 if (!use_limiter_) { 902 return true; 903 } 904 905 // Smoothly limit the mixed frame. 906 const int error = _limiter->ProcessStream(mixedAudio); 907 908 // And now we can safely restore the level. This procedure results in 909 // some loss of resolution, deemed acceptable. 910 // 911 // It's possible to apply the gain in the AGC (with a target level of 0 dbFS 912 // and compression gain of 6 dB). However, in the transition frame when this 913 // is enabled (moving from one to two participants) it has the potential to 914 // create discontinuities in the mixed frame. 915 // 916 // Instead we double the frame (with addition since left-shifting a 917 // negative value is undefined). 918 *mixedAudio += *mixedAudio; 919 920 if(error != _limiter->kNoError) { 921 WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id, 922 "Error from AudioProcessing: %d", error); 923 assert(false); 924 return false; 925 } 926 return true; 927 } 928 } // namespace webrtc 929
