Home | History | Annotate | Download | only in source 
      1 /*
      2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/modules/audio_conference_mixer/interface/audio_conference_mixer_defines.h"
     12 #include "webrtc/modules/audio_conference_mixer/source/audio_conference_mixer_impl.h"
     13 #include "webrtc/modules/audio_conference_mixer/source/audio_frame_manipulator.h"
     14 #include "webrtc/modules/audio_processing/include/audio_processing.h"
     15 #include "webrtc/modules/utility/interface/audio_frame_operations.h"
     16 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
     17 #include "webrtc/system_wrappers/interface/trace.h"
     18 
     19 namespace webrtc {
     20 namespace {
     21 
     22 struct ParticipantFramePair {
     23   MixerParticipant* participant;
     24   AudioFrame* audioFrame;
     25 };
     26 
     27 typedef std::list<ParticipantFramePair*> ParticipantFramePairList;
     28 
     29 // Mix |frame| into |mixed_frame|, with saturation protection and upmixing.
     30 // These effects are applied to |frame| itself prior to mixing. Assumes that
     31 // |mixed_frame| always has at least as many channels as |frame|. Supports
     32 // stereo at most.
     33 //
     34 // TODO(andrew): consider not modifying |frame| here.
     35 void MixFrames(AudioFrame* mixed_frame, AudioFrame* frame) {
     36   assert(mixed_frame->num_channels_ >= frame->num_channels_);
     37   // Divide by two to avoid saturation in the mixing.
     38   *frame >>= 1;
     39   if (mixed_frame->num_channels_ > frame->num_channels_) {
     40     // We only support mono-to-stereo.
     41     assert(mixed_frame->num_channels_ == 2 &&
     42            frame->num_channels_ == 1);
     43     AudioFrameOperations::MonoToStereo(frame);
     44   }
     45 
     46   *mixed_frame += *frame;
     47 }
     48 
     49 // Return the max number of channels from a |list| composed of AudioFrames.
     50 int MaxNumChannels(const AudioFrameList* list) {
     51   int max_num_channels = 1;
     52   for (AudioFrameList::const_iterator iter = list->begin();
     53        iter != list->end();
     54        ++iter) {
     55     max_num_channels = std::max(max_num_channels, (*iter)->num_channels_);
     56   }
     57   return max_num_channels;
     58 }
     59 
     60 void SetParticipantStatistics(ParticipantStatistics* stats,
     61                               const AudioFrame& frame) {
     62     stats->participant = frame.id_;
     63     stats->level = 0;  // TODO(andrew): to what should this be set?
     64 }
     65 
     66 }  // namespace
     67 
     68 MixerParticipant::MixerParticipant()
     69     : _mixHistory(new MixHistory()) {
     70 }
     71 
     72 MixerParticipant::~MixerParticipant() {
     73     delete _mixHistory;
     74 }
     75 
     76 int32_t MixerParticipant::IsMixed(bool& mixed) const {
     77     return _mixHistory->IsMixed(mixed);
     78 }
     79 
     80 MixHistory::MixHistory()
     81     : _isMixed(0) {
     82 }
     83 
     84 MixHistory::~MixHistory() {
     85 }
     86 
     87 int32_t MixHistory::IsMixed(bool& mixed) const {
     88     mixed = _isMixed;
     89     return 0;
     90 }
     91 
     92 int32_t MixHistory::WasMixed(bool& wasMixed) const {
     93     // Was mixed is the same as is mixed depending on perspective. This function
     94     // is for the perspective of AudioConferenceMixerImpl.
     95     return IsMixed(wasMixed);
     96 }
     97 
     98 int32_t MixHistory::SetIsMixed(const bool mixed) {
     99     _isMixed = mixed;
    100     return 0;
    101 }
    102 
    103 void MixHistory::ResetMixedStatus() {
    104     _isMixed = false;
    105 }
    106 
    107 AudioConferenceMixer* AudioConferenceMixer::Create(int id) {
    108     AudioConferenceMixerImpl* mixer = new AudioConferenceMixerImpl(id);
    109     if(!mixer->Init()) {
    110         delete mixer;
    111         return NULL;
    112     }
    113     return mixer;
    114 }
    115 
    116 AudioConferenceMixerImpl::AudioConferenceMixerImpl(int id)
    117     : _scratchParticipantsToMixAmount(0),
    118       _scratchMixedParticipants(),
    119       _scratchVadPositiveParticipantsAmount(0),
    120       _scratchVadPositiveParticipants(),
    121       _id(id),
    122       _minimumMixingFreq(kLowestPossible),
    123       _mixReceiver(NULL),
    124       _mixerStatusCallback(NULL),
    125       _amountOf10MsBetweenCallbacks(1),
    126       _amountOf10MsUntilNextCallback(0),
    127       _mixerStatusCb(false),
    128       _outputFrequency(kDefaultFrequency),
    129       _sampleSize(0),
    130       _audioFramePool(NULL),
    131       _participantList(),
    132       _additionalParticipantList(),
    133       _numMixedParticipants(0),
    134       _timeStamp(0),
    135       _timeScheduler(kProcessPeriodicityInMs),
    136       _mixedAudioLevel(),
    137       _processCalls(0) {}
    138 
    139 bool AudioConferenceMixerImpl::Init() {
    140     _crit.reset(CriticalSectionWrapper::CreateCriticalSection());
    141     if (_crit.get() == NULL)
    142         return false;
    143 
    144     _cbCrit.reset(CriticalSectionWrapper::CreateCriticalSection());
    145     if(_cbCrit.get() == NULL)
    146         return false;
    147 
    148     Config config;
    149     config.Set<ExperimentalAgc>(new ExperimentalAgc(false));
    150     _limiter.reset(AudioProcessing::Create(config));
    151     if(!_limiter.get())
    152         return false;
    153 
    154     MemoryPool<AudioFrame>::CreateMemoryPool(_audioFramePool,
    155                                              DEFAULT_AUDIO_FRAME_POOLSIZE);
    156     if(_audioFramePool == NULL)
    157         return false;
    158 
    159     if(SetOutputFrequency(kDefaultFrequency) == -1)
    160         return false;
    161 
    162     if(_limiter->gain_control()->set_mode(GainControl::kFixedDigital) !=
    163         _limiter->kNoError)
    164         return false;
    165 
    166     // We smoothly limit the mixed frame to -7 dbFS. -6 would correspond to the
    167     // divide-by-2 but -7 is used instead to give a bit of headroom since the
    168     // AGC is not a hard limiter.
    169     if(_limiter->gain_control()->set_target_level_dbfs(7) != _limiter->kNoError)
    170         return false;
    171 
    172     if(_limiter->gain_control()->set_compression_gain_db(0)
    173         != _limiter->kNoError)
    174         return false;
    175 
    176     if(_limiter->gain_control()->enable_limiter(true) != _limiter->kNoError)
    177         return false;
    178 
    179     if(_limiter->gain_control()->Enable(true) != _limiter->kNoError)
    180         return false;
    181 
    182     return true;
    183 }
    184 
    185 AudioConferenceMixerImpl::~AudioConferenceMixerImpl() {
    186     MemoryPool<AudioFrame>::DeleteMemoryPool(_audioFramePool);
    187     assert(_audioFramePool == NULL);
    188 }
    189 
    190 int32_t AudioConferenceMixerImpl::ChangeUniqueId(const int32_t id) {
    191     _id = id;
    192     return 0;
    193 }
    194 
    195 // Process should be called every kProcessPeriodicityInMs ms
    196 int32_t AudioConferenceMixerImpl::TimeUntilNextProcess() {
    197     int32_t timeUntilNextProcess = 0;
    198     CriticalSectionScoped cs(_crit.get());
    199     if(_timeScheduler.TimeToNextUpdate(timeUntilNextProcess) != 0) {
    200         WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
    201                      "failed in TimeToNextUpdate() call");
    202         // Sanity check
    203         assert(false);
    204         return -1;
    205     }
    206     return timeUntilNextProcess;
    207 }
    208 
    209 int32_t AudioConferenceMixerImpl::Process() {
    210     size_t remainingParticipantsAllowedToMix =
    211         kMaximumAmountOfMixedParticipants;
    212     {
    213         CriticalSectionScoped cs(_crit.get());
    214         assert(_processCalls == 0);
    215         _processCalls++;
    216 
    217         // Let the scheduler know that we are running one iteration.
    218         _timeScheduler.UpdateScheduler();
    219     }
    220 
    221     AudioFrameList mixList;
    222     AudioFrameList rampOutList;
    223     AudioFrameList additionalFramesList;
    224     std::map<int, MixerParticipant*> mixedParticipantsMap;
    225     {
    226         CriticalSectionScoped cs(_cbCrit.get());
    227 
    228         int32_t lowFreq = GetLowestMixingFrequency();
    229         // SILK can run in 12 kHz and 24 kHz. These frequencies are not
    230         // supported so use the closest higher frequency to not lose any
    231         // information.
    232         // TODO(henrike): this is probably more appropriate to do in
    233         //                GetLowestMixingFrequency().
    234         if (lowFreq == 12000) {
    235             lowFreq = 16000;
    236         } else if (lowFreq == 24000) {
    237             lowFreq = 32000;
    238         }
    239         if(lowFreq <= 0) {
    240             CriticalSectionScoped cs(_crit.get());
    241             _processCalls--;
    242             return 0;
    243         } else {
    244             switch(lowFreq) {
    245             case 8000:
    246                 if(OutputFrequency() != kNbInHz) {
    247                     SetOutputFrequency(kNbInHz);
    248                 }
    249                 break;
    250             case 16000:
    251                 if(OutputFrequency() != kWbInHz) {
    252                     SetOutputFrequency(kWbInHz);
    253                 }
    254                 break;
    255             case 32000:
    256                 if(OutputFrequency() != kSwbInHz) {
    257                     SetOutputFrequency(kSwbInHz);
    258                 }
    259                 break;
    260             case 48000:
    261                 if(OutputFrequency() != kFbInHz) {
    262                     SetOutputFrequency(kFbInHz);
    263                 }
    264                 break;
    265             default:
    266                 assert(false);
    267 
    268                 CriticalSectionScoped cs(_crit.get());
    269                 _processCalls--;
    270                 return -1;
    271             }
    272         }
    273 
    274         UpdateToMix(&mixList, &rampOutList, &mixedParticipantsMap,
    275                     remainingParticipantsAllowedToMix);
    276 
    277         GetAdditionalAudio(&additionalFramesList);
    278         UpdateMixedStatus(mixedParticipantsMap);
    279         _scratchParticipantsToMixAmount = mixedParticipantsMap.size();
    280     }
    281 
    282     // Get an AudioFrame for mixing from the memory pool.
    283     AudioFrame* mixedAudio = NULL;
    284     if(_audioFramePool->PopMemory(mixedAudio) == -1) {
    285         WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id,
    286                      "failed PopMemory() call");
    287         assert(false);
    288         return -1;
    289     }
    290 
    291     bool timeForMixerCallback = false;
    292     int retval = 0;
    293     int32_t audioLevel = 0;
    294     {
    295         CriticalSectionScoped cs(_crit.get());
    296 
    297         // TODO(henrike): it might be better to decide the number of channels
    298         //                with an API instead of dynamically.
    299 
    300         // Find the max channels over all mixing lists.
    301         const int num_mixed_channels = std::max(MaxNumChannels(&mixList),
    302             std::max(MaxNumChannels(&additionalFramesList),
    303                      MaxNumChannels(&rampOutList)));
    304 
    305         mixedAudio->UpdateFrame(-1, _timeStamp, NULL, 0, _outputFrequency,
    306                                 AudioFrame::kNormalSpeech,
    307                                 AudioFrame::kVadPassive, num_mixed_channels);
    308 
    309         _timeStamp += _sampleSize;
    310 
    311         MixFromList(*mixedAudio, &mixList);
    312         MixAnonomouslyFromList(*mixedAudio, &additionalFramesList);
    313         MixAnonomouslyFromList(*mixedAudio, &rampOutList);
    314 
    315         if(mixedAudio->samples_per_channel_ == 0) {
    316             // Nothing was mixed, set the audio samples to silence.
    317             mixedAudio->samples_per_channel_ = _sampleSize;
    318             mixedAudio->Mute();
    319         } else {
    320             // Only call the limiter if we have something to mix.
    321             if(!LimitMixedAudio(*mixedAudio))
    322                 retval = -1;
    323         }
    324 
    325         _mixedAudioLevel.ComputeLevel(mixedAudio->data_,_sampleSize);
    326         audioLevel = _mixedAudioLevel.GetLevel();
    327 
    328         if(_mixerStatusCb) {
    329             _scratchVadPositiveParticipantsAmount = 0;
    330             UpdateVADPositiveParticipants(&mixList);
    331             if(_amountOf10MsUntilNextCallback-- == 0) {
    332                 _amountOf10MsUntilNextCallback = _amountOf10MsBetweenCallbacks;
    333                 timeForMixerCallback = true;
    334             }
    335         }
    336     }
    337 
    338     {
    339         CriticalSectionScoped cs(_cbCrit.get());
    340         if(_mixReceiver != NULL) {
    341             const AudioFrame** dummy = NULL;
    342             _mixReceiver->NewMixedAudio(
    343                 _id,
    344                 *mixedAudio,
    345                 dummy,
    346                 0);
    347         }
    348 
    349         if((_mixerStatusCallback != NULL) &&
    350             timeForMixerCallback) {
    351             _mixerStatusCallback->MixedParticipants(
    352                 _id,
    353                 _scratchMixedParticipants,
    354                 static_cast<uint32_t>(_scratchParticipantsToMixAmount));
    355 
    356             _mixerStatusCallback->VADPositiveParticipants(
    357                 _id,
    358                 _scratchVadPositiveParticipants,
    359                 _scratchVadPositiveParticipantsAmount);
    360             _mixerStatusCallback->MixedAudioLevel(_id,audioLevel);
    361         }
    362     }
    363 
    364     // Reclaim all outstanding memory.
    365     _audioFramePool->PushMemory(mixedAudio);
    366     ClearAudioFrameList(&mixList);
    367     ClearAudioFrameList(&rampOutList);
    368     ClearAudioFrameList(&additionalFramesList);
    369     {
    370         CriticalSectionScoped cs(_crit.get());
    371         _processCalls--;
    372     }
    373     return retval;
    374 }
    375 
    376 int32_t AudioConferenceMixerImpl::RegisterMixedStreamCallback(
    377     AudioMixerOutputReceiver& mixReceiver) {
    378     CriticalSectionScoped cs(_cbCrit.get());
    379     if(_mixReceiver != NULL) {
    380         return -1;
    381     }
    382     _mixReceiver = &mixReceiver;
    383     return 0;
    384 }
    385 
    386 int32_t AudioConferenceMixerImpl::UnRegisterMixedStreamCallback() {
    387     CriticalSectionScoped cs(_cbCrit.get());
    388     if(_mixReceiver == NULL) {
    389         return -1;
    390     }
    391     _mixReceiver = NULL;
    392     return 0;
    393 }
    394 
    395 int32_t AudioConferenceMixerImpl::SetOutputFrequency(
    396     const Frequency frequency) {
    397     CriticalSectionScoped cs(_crit.get());
    398 
    399     _outputFrequency = frequency;
    400     _sampleSize = (_outputFrequency*kProcessPeriodicityInMs) / 1000;
    401 
    402     return 0;
    403 }
    404 
    405 AudioConferenceMixer::Frequency
    406 AudioConferenceMixerImpl::OutputFrequency() const {
    407     CriticalSectionScoped cs(_crit.get());
    408     return _outputFrequency;
    409 }
    410 
    411 int32_t AudioConferenceMixerImpl::RegisterMixerStatusCallback(
    412     AudioMixerStatusReceiver& mixerStatusCallback,
    413     const uint32_t amountOf10MsBetweenCallbacks) {
    414     if(amountOf10MsBetweenCallbacks == 0) {
    415         WEBRTC_TRACE(
    416             kTraceWarning,
    417             kTraceAudioMixerServer,
    418             _id,
    419             "amountOf10MsBetweenCallbacks(%d) needs to be larger than 0");
    420         return -1;
    421     }
    422     {
    423         CriticalSectionScoped cs(_cbCrit.get());
    424         if(_mixerStatusCallback != NULL) {
    425             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
    426                          "Mixer status callback already registered");
    427             return -1;
    428         }
    429         _mixerStatusCallback = &mixerStatusCallback;
    430     }
    431     {
    432         CriticalSectionScoped cs(_crit.get());
    433         _amountOf10MsBetweenCallbacks  = amountOf10MsBetweenCallbacks;
    434         _amountOf10MsUntilNextCallback = 0;
    435         _mixerStatusCb                 = true;
    436     }
    437     return 0;
    438 }
    439 
    440 int32_t AudioConferenceMixerImpl::UnRegisterMixerStatusCallback() {
    441     {
    442         CriticalSectionScoped cs(_crit.get());
    443         if(!_mixerStatusCb)
    444         {
    445             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
    446                          "Mixer status callback not registered");
    447             return -1;
    448         }
    449         _mixerStatusCb = false;
    450     }
    451     {
    452         CriticalSectionScoped cs(_cbCrit.get());
    453         _mixerStatusCallback = NULL;
    454     }
    455     return 0;
    456 }
    457 
    458 int32_t AudioConferenceMixerImpl::SetMixabilityStatus(
    459     MixerParticipant& participant,
    460     bool mixable) {
    461     if (!mixable) {
    462         // Anonymous participants are in a separate list. Make sure that the
    463         // participant is in the _participantList if it is being mixed.
    464         SetAnonymousMixabilityStatus(participant, false);
    465     }
    466     size_t numMixedParticipants;
    467     {
    468         CriticalSectionScoped cs(_cbCrit.get());
    469         const bool isMixed =
    470             IsParticipantInList(participant, &_participantList);
    471         // API must be called with a new state.
    472         if(!(mixable ^ isMixed)) {
    473             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
    474                          "Mixable is aready %s",
    475                          isMixed ? "ON" : "off");
    476             return -1;
    477         }
    478         bool success = false;
    479         if(mixable) {
    480             success = AddParticipantToList(participant, &_participantList);
    481         } else {
    482             success = RemoveParticipantFromList(participant, &_participantList);
    483         }
    484         if(!success) {
    485             WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
    486                          "failed to %s participant",
    487                          mixable ? "add" : "remove");
    488             assert(false);
    489             return -1;
    490         }
    491 
    492         size_t numMixedNonAnonymous = _participantList.size();
    493         if (numMixedNonAnonymous > kMaximumAmountOfMixedParticipants) {
    494             numMixedNonAnonymous = kMaximumAmountOfMixedParticipants;
    495         }
    496         numMixedParticipants =
    497             numMixedNonAnonymous + _additionalParticipantList.size();
    498     }
    499     // A MixerParticipant was added or removed. Make sure the scratch
    500     // buffer is updated if necessary.
    501     // Note: The scratch buffer may only be updated in Process().
    502     CriticalSectionScoped cs(_crit.get());
    503     _numMixedParticipants = numMixedParticipants;
    504     return 0;
    505 }
    506 
    507 int32_t AudioConferenceMixerImpl::MixabilityStatus(
    508     MixerParticipant& participant,
    509     bool& mixable) {
    510     CriticalSectionScoped cs(_cbCrit.get());
    511     mixable = IsParticipantInList(participant, &_participantList);
    512     return 0;
    513 }
    514 
    515 int32_t AudioConferenceMixerImpl::SetAnonymousMixabilityStatus(
    516     MixerParticipant& participant, const bool anonymous) {
    517     CriticalSectionScoped cs(_cbCrit.get());
    518     if(IsParticipantInList(participant, &_additionalParticipantList)) {
    519         if(anonymous) {
    520             return 0;
    521         }
    522         if(!RemoveParticipantFromList(participant,
    523                                       &_additionalParticipantList)) {
    524             WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
    525                          "unable to remove participant from anonymous list");
    526             assert(false);
    527             return -1;
    528         }
    529         return AddParticipantToList(participant, &_participantList) ? 0 : -1;
    530     }
    531     if(!anonymous) {
    532         return 0;
    533     }
    534     const bool mixable = RemoveParticipantFromList(participant,
    535                                                    &_participantList);
    536     if(!mixable) {
    537         WEBRTC_TRACE(
    538             kTraceWarning,
    539             kTraceAudioMixerServer,
    540             _id,
    541             "participant must be registered before turning it into anonymous");
    542         // Setting anonymous status is only possible if MixerParticipant is
    543         // already registered.
    544         return -1;
    545     }
    546     return AddParticipantToList(participant, &_additionalParticipantList) ?
    547         0 : -1;
    548 }
    549 
    550 int32_t AudioConferenceMixerImpl::AnonymousMixabilityStatus(
    551     MixerParticipant& participant, bool& mixable) {
    552     CriticalSectionScoped cs(_cbCrit.get());
    553     mixable = IsParticipantInList(participant,
    554                                   &_additionalParticipantList);
    555     return 0;
    556 }
    557 
    558 int32_t AudioConferenceMixerImpl::SetMinimumMixingFrequency(
    559     Frequency freq) {
    560     // Make sure that only allowed sampling frequencies are used. Use closest
    561     // higher sampling frequency to avoid losing information.
    562     if (static_cast<int>(freq) == 12000) {
    563          freq = kWbInHz;
    564     } else if (static_cast<int>(freq) == 24000) {
    565         freq = kSwbInHz;
    566     }
    567 
    568     if((freq == kNbInHz) || (freq == kWbInHz) || (freq == kSwbInHz) ||
    569        (freq == kLowestPossible)) {
    570         _minimumMixingFreq=freq;
    571         return 0;
    572     } else {
    573         WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
    574                      "SetMinimumMixingFrequency incorrect frequency: %i",freq);
    575         assert(false);
    576         return -1;
    577     }
    578 }
    579 
    580 // Check all AudioFrames that are to be mixed. The highest sampling frequency
    581 // found is the lowest that can be used without losing information.
    582 int32_t AudioConferenceMixerImpl::GetLowestMixingFrequency() {
    583     const int participantListFrequency =
    584         GetLowestMixingFrequencyFromList(&_participantList);
    585     const int anonymousListFrequency =
    586         GetLowestMixingFrequencyFromList(&_additionalParticipantList);
    587     const int highestFreq =
    588         (participantListFrequency > anonymousListFrequency) ?
    589             participantListFrequency : anonymousListFrequency;
    590     // Check if the user specified a lowest mixing frequency.
    591     if(_minimumMixingFreq != kLowestPossible) {
    592         if(_minimumMixingFreq > highestFreq) {
    593             return _minimumMixingFreq;
    594         }
    595     }
    596     return highestFreq;
    597 }
    598 
    599 int32_t AudioConferenceMixerImpl::GetLowestMixingFrequencyFromList(
    600     MixerParticipantList* mixList) {
    601     int32_t highestFreq = 8000;
    602     for (MixerParticipantList::iterator iter = mixList->begin();
    603          iter != mixList->end();
    604          ++iter) {
    605         const int32_t neededFrequency = (*iter)->NeededFrequency(_id);
    606         if(neededFrequency > highestFreq) {
    607             highestFreq = neededFrequency;
    608         }
    609     }
    610     return highestFreq;
    611 }
    612 
    613 void AudioConferenceMixerImpl::UpdateToMix(
    614     AudioFrameList* mixList,
    615     AudioFrameList* rampOutList,
    616     std::map<int, MixerParticipant*>* mixParticipantList,
    617     size_t& maxAudioFrameCounter) {
    618     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
    619                  "UpdateToMix(mixList,rampOutList,mixParticipantList,%d)",
    620                  maxAudioFrameCounter);
    621     const size_t mixListStartSize = mixList->size();
    622     AudioFrameList activeList;
    623     // Struct needed by the passive lists to keep track of which AudioFrame
    624     // belongs to which MixerParticipant.
    625     ParticipantFramePairList passiveWasNotMixedList;
    626     ParticipantFramePairList passiveWasMixedList;
    627     for (MixerParticipantList::iterator participant = _participantList.begin();
    628          participant != _participantList.end();
    629          ++participant) {
    630         // Stop keeping track of passive participants if there are already
    631         // enough participants available (they wont be mixed anyway).
    632         bool mustAddToPassiveList = (maxAudioFrameCounter >
    633                                     (activeList.size() +
    634                                      passiveWasMixedList.size() +
    635                                      passiveWasNotMixedList.size()));
    636 
    637         bool wasMixed = false;
    638         (*participant)->_mixHistory->WasMixed(wasMixed);
    639         AudioFrame* audioFrame = NULL;
    640         if(_audioFramePool->PopMemory(audioFrame) == -1) {
    641             WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id,
    642                          "failed PopMemory() call");
    643             assert(false);
    644             return;
    645         }
    646         audioFrame->sample_rate_hz_ = _outputFrequency;
    647 
    648         if((*participant)->GetAudioFrame(_id,*audioFrame) != 0) {
    649             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
    650                          "failed to GetAudioFrame() from participant");
    651             _audioFramePool->PushMemory(audioFrame);
    652             continue;
    653         }
    654         if (_participantList.size() != 1) {
    655           // TODO(wu): Issue 3390, add support for multiple participants case.
    656           audioFrame->ntp_time_ms_ = -1;
    657         }
    658 
    659         // TODO(henrike): this assert triggers in some test cases where SRTP is
    660         // used which prevents NetEQ from making a VAD. Temporarily disable this
    661         // assert until the problem is fixed on a higher level.
    662         // assert(audioFrame->vad_activity_ != AudioFrame::kVadUnknown);
    663         if (audioFrame->vad_activity_ == AudioFrame::kVadUnknown) {
    664             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
    665                          "invalid VAD state from participant");
    666         }
    667 
    668         if(audioFrame->vad_activity_ == AudioFrame::kVadActive) {
    669             if(!wasMixed) {
    670                 RampIn(*audioFrame);
    671             }
    672 
    673             if(activeList.size() >= maxAudioFrameCounter) {
    674                 // There are already more active participants than should be
    675                 // mixed. Only keep the ones with the highest energy.
    676                 AudioFrameList::iterator replaceItem;
    677                 CalculateEnergy(*audioFrame);
    678                 uint32_t lowestEnergy = audioFrame->energy_;
    679 
    680                 bool found_replace_item = false;
    681                 for (AudioFrameList::iterator iter = activeList.begin();
    682                      iter != activeList.end();
    683                      ++iter) {
    684                     CalculateEnergy(**iter);
    685                     if((*iter)->energy_ < lowestEnergy) {
    686                         replaceItem = iter;
    687                         lowestEnergy = (*iter)->energy_;
    688                         found_replace_item = true;
    689                     }
    690                 }
    691                 if(found_replace_item) {
    692                     AudioFrame* replaceFrame = *replaceItem;
    693 
    694                     bool replaceWasMixed = false;
    695                     std::map<int, MixerParticipant*>::iterator it =
    696                         mixParticipantList->find(replaceFrame->id_);
    697 
    698                     // When a frame is pushed to |activeList| it is also pushed
    699                     // to mixParticipantList with the frame's id. This means
    700                     // that the Find call above should never fail.
    701                     assert(it != mixParticipantList->end());
    702                     it->second->_mixHistory->WasMixed(replaceWasMixed);
    703 
    704                     mixParticipantList->erase(replaceFrame->id_);
    705                     activeList.erase(replaceItem);
    706 
    707                     activeList.push_front(audioFrame);
    708                     (*mixParticipantList)[audioFrame->id_] = *participant;
    709                     assert(mixParticipantList->size() <=
    710                            kMaximumAmountOfMixedParticipants);
    711 
    712                     if (replaceWasMixed) {
    713                       RampOut(*replaceFrame);
    714                       rampOutList->push_back(replaceFrame);
    715                       assert(rampOutList->size() <=
    716                              kMaximumAmountOfMixedParticipants);
    717                     } else {
    718                       _audioFramePool->PushMemory(replaceFrame);
    719                     }
    720                 } else {
    721                     if(wasMixed) {
    722                         RampOut(*audioFrame);
    723                         rampOutList->push_back(audioFrame);
    724                         assert(rampOutList->size() <=
    725                                kMaximumAmountOfMixedParticipants);
    726                     } else {
    727                         _audioFramePool->PushMemory(audioFrame);
    728                     }
    729                 }
    730             } else {
    731                 activeList.push_front(audioFrame);
    732                 (*mixParticipantList)[audioFrame->id_] = *participant;
    733                 assert(mixParticipantList->size() <=
    734                        kMaximumAmountOfMixedParticipants);
    735             }
    736         } else {
    737             if(wasMixed) {
    738                 ParticipantFramePair* pair = new ParticipantFramePair;
    739                 pair->audioFrame  = audioFrame;
    740                 pair->participant = *participant;
    741                 passiveWasMixedList.push_back(pair);
    742             } else if(mustAddToPassiveList) {
    743                 RampIn(*audioFrame);
    744                 ParticipantFramePair* pair = new ParticipantFramePair;
    745                 pair->audioFrame  = audioFrame;
    746                 pair->participant = *participant;
    747                 passiveWasNotMixedList.push_back(pair);
    748             } else {
    749                 _audioFramePool->PushMemory(audioFrame);
    750             }
    751         }
    752     }
    753     assert(activeList.size() <= maxAudioFrameCounter);
    754     // At this point it is known which participants should be mixed. Transfer
    755     // this information to this functions output parameters.
    756     for (AudioFrameList::iterator iter = activeList.begin();
    757          iter != activeList.end();
    758          ++iter) {
    759         mixList->push_back(*iter);
    760     }
    761     activeList.clear();
    762     // Always mix a constant number of AudioFrames. If there aren't enough
    763     // active participants mix passive ones. Starting with those that was mixed
    764     // last iteration.
    765     for (ParticipantFramePairList::iterator iter = passiveWasMixedList.begin();
    766          iter != passiveWasMixedList.end();
    767          ++iter) {
    768         if(mixList->size() < maxAudioFrameCounter + mixListStartSize) {
    769             mixList->push_back((*iter)->audioFrame);
    770             (*mixParticipantList)[(*iter)->audioFrame->id_] =
    771                 (*iter)->participant;
    772             assert(mixParticipantList->size() <=
    773                    kMaximumAmountOfMixedParticipants);
    774         } else {
    775             _audioFramePool->PushMemory((*iter)->audioFrame);
    776         }
    777         delete *iter;
    778     }
    779     // And finally the ones that have not been mixed for a while.
    780     for (ParticipantFramePairList::iterator iter =
    781              passiveWasNotMixedList.begin();
    782          iter != passiveWasNotMixedList.end();
    783          ++iter) {
    784         if(mixList->size() <  maxAudioFrameCounter + mixListStartSize) {
    785           mixList->push_back((*iter)->audioFrame);
    786             (*mixParticipantList)[(*iter)->audioFrame->id_] =
    787                 (*iter)->participant;
    788             assert(mixParticipantList->size() <=
    789                    kMaximumAmountOfMixedParticipants);
    790         } else {
    791             _audioFramePool->PushMemory((*iter)->audioFrame);
    792         }
    793         delete *iter;
    794     }
    795     assert(maxAudioFrameCounter + mixListStartSize >= mixList->size());
    796     maxAudioFrameCounter += mixListStartSize - mixList->size();
    797 }
    798 
    799 void AudioConferenceMixerImpl::GetAdditionalAudio(
    800     AudioFrameList* additionalFramesList) {
    801     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
    802                  "GetAdditionalAudio(additionalFramesList)");
    803     // The GetAudioFrame() callback may result in the participant being removed
    804     // from additionalParticipantList_. If that happens it will invalidate any
    805     // iterators. Create a copy of the participants list such that the list of
    806     // participants can be traversed safely.
    807     MixerParticipantList additionalParticipantList;
    808     additionalParticipantList.insert(additionalParticipantList.begin(),
    809                                      _additionalParticipantList.begin(),
    810                                      _additionalParticipantList.end());
    811 
    812     for (MixerParticipantList::iterator participant =
    813              additionalParticipantList.begin();
    814          participant != additionalParticipantList.end();
    815          ++participant) {
    816         AudioFrame* audioFrame = NULL;
    817         if(_audioFramePool->PopMemory(audioFrame) == -1) {
    818             WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id,
    819                          "failed PopMemory() call");
    820             assert(false);
    821             return;
    822         }
    823         audioFrame->sample_rate_hz_ = _outputFrequency;
    824         if((*participant)->GetAudioFrame(_id, *audioFrame) != 0) {
    825             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
    826                          "failed to GetAudioFrame() from participant");
    827             _audioFramePool->PushMemory(audioFrame);
    828             continue;
    829         }
    830         if(audioFrame->samples_per_channel_ == 0) {
    831             // Empty frame. Don't use it.
    832             _audioFramePool->PushMemory(audioFrame);
    833             continue;
    834         }
    835         additionalFramesList->push_back(audioFrame);
    836     }
    837 }
    838 
    839 void AudioConferenceMixerImpl::UpdateMixedStatus(
    840     std::map<int, MixerParticipant*>& mixedParticipantsMap) {
    841     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
    842                  "UpdateMixedStatus(mixedParticipantsMap)");
    843     assert(mixedParticipantsMap.size() <= kMaximumAmountOfMixedParticipants);
    844 
    845     // Loop through all participants. If they are in the mix map they
    846     // were mixed.
    847     for (MixerParticipantList::iterator participant = _participantList.begin();
    848          participant != _participantList.end();
    849          ++participant) {
    850         bool isMixed = false;
    851         for (std::map<int, MixerParticipant*>::iterator it =
    852                  mixedParticipantsMap.begin();
    853              it != mixedParticipantsMap.end();
    854              ++it) {
    855           if (it->second == *participant) {
    856             isMixed = true;
    857             break;
    858           }
    859         }
    860         (*participant)->_mixHistory->SetIsMixed(isMixed);
    861     }
    862 }
    863 
    864 void AudioConferenceMixerImpl::ClearAudioFrameList(
    865     AudioFrameList* audioFrameList) {
    866     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
    867                  "ClearAudioFrameList(audioFrameList)");
    868     for (AudioFrameList::iterator iter = audioFrameList->begin();
    869          iter != audioFrameList->end();
    870          ++iter) {
    871         _audioFramePool->PushMemory(*iter);
    872     }
    873     audioFrameList->clear();
    874 }
    875 
    876 void AudioConferenceMixerImpl::UpdateVADPositiveParticipants(
    877     AudioFrameList* mixList) {
    878     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
    879                  "UpdateVADPositiveParticipants(mixList)");
    880 
    881     for (AudioFrameList::iterator iter = mixList->begin();
    882          iter != mixList->end();
    883          ++iter) {
    884         CalculateEnergy(**iter);
    885         if((*iter)->vad_activity_ == AudioFrame::kVadActive) {
    886             _scratchVadPositiveParticipants[
    887                 _scratchVadPositiveParticipantsAmount].participant =
    888                 (*iter)->id_;
    889             // TODO(andrew): to what should this be set?
    890             _scratchVadPositiveParticipants[
    891                 _scratchVadPositiveParticipantsAmount].level = 0;
    892             _scratchVadPositiveParticipantsAmount++;
    893         }
    894     }
    895 }
    896 
    897 bool AudioConferenceMixerImpl::IsParticipantInList(
    898     MixerParticipant& participant,
    899     MixerParticipantList* participantList) const {
    900     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
    901                  "IsParticipantInList(participant,participantList)");
    902     for (MixerParticipantList::const_iterator iter = participantList->begin();
    903          iter != participantList->end();
    904          ++iter) {
    905         if(&participant == *iter) {
    906             return true;
    907         }
    908     }
    909     return false;
    910 }
    911 
    912 bool AudioConferenceMixerImpl::AddParticipantToList(
    913     MixerParticipant& participant,
    914     MixerParticipantList* participantList) {
    915     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
    916                  "AddParticipantToList(participant, participantList)");
    917     participantList->push_back(&participant);
    918     // Make sure that the mixed status is correct for new MixerParticipant.
    919     participant._mixHistory->ResetMixedStatus();
    920     return true;
    921 }
    922 
    923 bool AudioConferenceMixerImpl::RemoveParticipantFromList(
    924     MixerParticipant& participant,
    925     MixerParticipantList* participantList) {
    926     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
    927                  "RemoveParticipantFromList(participant, participantList)");
    928     for (MixerParticipantList::iterator iter = participantList->begin();
    929          iter != participantList->end();
    930          ++iter) {
    931         if(*iter == &participant) {
    932             participantList->erase(iter);
    933             // Participant is no longer mixed, reset to default.
    934             participant._mixHistory->ResetMixedStatus();
    935             return true;
    936         }
    937     }
    938     return false;
    939 }
    940 
    941 int32_t AudioConferenceMixerImpl::MixFromList(
    942     AudioFrame& mixedAudio,
    943     const AudioFrameList* audioFrameList) {
    944     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
    945                  "MixFromList(mixedAudio, audioFrameList)");
    946     if(audioFrameList->empty()) return 0;
    947 
    948     uint32_t position = 0;
    949     if(_numMixedParticipants == 1) {
    950         // No mixing required here; skip the saturation protection.
    951         AudioFrame* audioFrame = audioFrameList->front();
    952         mixedAudio.CopyFrom(*audioFrame);
    953         SetParticipantStatistics(&_scratchMixedParticipants[position],
    954                                  *audioFrame);
    955         return 0;
    956     }
    957 
    958     if (audioFrameList->size() == 1) {
    959       mixedAudio.timestamp_ = audioFrameList->front()->timestamp_;
    960       mixedAudio.elapsed_time_ms_ = audioFrameList->front()->elapsed_time_ms_;
    961     } else {
    962       // TODO(wu): Issue 3390.
    963       // Audio frame timestamp is only supported in one channel case.
    964       mixedAudio.timestamp_ = 0;
    965       mixedAudio.elapsed_time_ms_ = -1;
    966     }
    967 
    968     for (AudioFrameList::const_iterator iter = audioFrameList->begin();
    969          iter != audioFrameList->end();
    970          ++iter) {
    971         if(position >= kMaximumAmountOfMixedParticipants) {
    972             WEBRTC_TRACE(
    973                 kTraceMemory,
    974                 kTraceAudioMixerServer,
    975                 _id,
    976                 "Trying to mix more than max amount of mixed participants:%d!",
    977                 kMaximumAmountOfMixedParticipants);
    978             // Assert and avoid crash
    979             assert(false);
    980             position = 0;
    981         }
    982         MixFrames(&mixedAudio, (*iter));
    983 
    984         SetParticipantStatistics(&_scratchMixedParticipants[position],
    985                                  **iter);
    986 
    987         position++;
    988     }
    989 
    990     return 0;
    991 }
    992 
    993 // TODO(andrew): consolidate this function with MixFromList.
    994 int32_t AudioConferenceMixerImpl::MixAnonomouslyFromList(
    995     AudioFrame& mixedAudio,
    996     const AudioFrameList* audioFrameList) {
    997     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
    998                  "MixAnonomouslyFromList(mixedAudio, audioFrameList)");
    999 
   1000     if(audioFrameList->empty()) return 0;
   1001 
   1002     if(_numMixedParticipants == 1) {
   1003         // No mixing required here; skip the saturation protection.
   1004         AudioFrame* audioFrame = audioFrameList->front();
   1005         mixedAudio.CopyFrom(*audioFrame);
   1006         return 0;
   1007     }
   1008 
   1009     for (AudioFrameList::const_iterator iter = audioFrameList->begin();
   1010          iter != audioFrameList->end();
   1011          ++iter) {
   1012         MixFrames(&mixedAudio, *iter);
   1013     }
   1014     return 0;
   1015 }
   1016 
   1017 bool AudioConferenceMixerImpl::LimitMixedAudio(AudioFrame& mixedAudio) {
   1018     if(_numMixedParticipants == 1) {
   1019         return true;
   1020     }
   1021 
   1022     // Smoothly limit the mixed frame.
   1023     const int error = _limiter->ProcessStream(&mixedAudio);
   1024 
   1025     // And now we can safely restore the level. This procedure results in
   1026     // some loss of resolution, deemed acceptable.
   1027     //
   1028     // It's possible to apply the gain in the AGC (with a target level of 0 dbFS
   1029     // and compression gain of 6 dB). However, in the transition frame when this
   1030     // is enabled (moving from one to two participants) it has the potential to
   1031     // create discontinuities in the mixed frame.
   1032     //
   1033     // Instead we double the frame (with addition since left-shifting a
   1034     // negative value is undefined).
   1035     mixedAudio += mixedAudio;
   1036 
   1037     if(error != _limiter->kNoError) {
   1038         WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
   1039                      "Error from AudioProcessing: %d", error);
   1040         assert(false);
   1041         return false;
   1042     }
   1043     return true;
   1044 }
   1045 }  // namespace webrtc
   1046