Home | History | Annotate | Download | only in android 
      1 /*
      2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/modules/audio_device/android/opensles_output.h"
     12 
     13 #include <assert.h>
     14 
     15 #include "webrtc/modules/audio_device/android/opensles_common.h"
     16 #include "webrtc/modules/audio_device/android/fine_audio_buffer.h"
     17 #include "webrtc/modules/audio_device/android/single_rw_fifo.h"
     18 #include "webrtc/modules/audio_device/audio_device_buffer.h"
     19 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
     20 #include "webrtc/system_wrappers/interface/thread_wrapper.h"
     21 #include "webrtc/system_wrappers/interface/trace.h"
     22 
     23 #define VOID_RETURN
     24 #define OPENSL_RETURN_ON_FAILURE(op, ret_val)                    \
     25   do {                                                           \
     26     SLresult err = (op);                                         \
     27     if (err != SL_RESULT_SUCCESS) {                              \
     28       WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_,          \
     29                    "OpenSL error: %d", err);                     \
     30       assert(false);                                             \
     31       return ret_val;                                            \
     32     }                                                            \
     33   } while (0)
     34 
     35 static const SLEngineOption kOption[] = {
     36   { SL_ENGINEOPTION_THREADSAFE, static_cast<SLuint32>(SL_BOOLEAN_TRUE) },
     37 };
     38 
     39 enum {
     40   kNoUnderrun,
     41   kUnderrun,
     42 };
     43 
     44 namespace webrtc {
     45 
     46 OpenSlesOutput::OpenSlesOutput(const int32_t id)
     47     : id_(id),
     48       initialized_(false),
     49       speaker_initialized_(false),
     50       play_initialized_(false),
     51       crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
     52       playing_(false),
     53       num_fifo_buffers_needed_(0),
     54       number_underruns_(0),
     55       sles_engine_(NULL),
     56       sles_engine_itf_(NULL),
     57       sles_player_(NULL),
     58       sles_player_itf_(NULL),
     59       sles_player_sbq_itf_(NULL),
     60       sles_output_mixer_(NULL),
     61       audio_buffer_(NULL),
     62       active_queue_(0),
     63       speaker_sampling_rate_(kDefaultSampleRate),
     64       buffer_size_samples_(0),
     65       buffer_size_bytes_(0),
     66       playout_delay_(0) {
     67 }
     68 
     69 OpenSlesOutput::~OpenSlesOutput() {
     70 }
     71 
     72 int32_t OpenSlesOutput::SetAndroidAudioDeviceObjects(void* javaVM,
     73                                                      void* env,
     74                                                      void* context) {
     75   AudioManagerJni::SetAndroidAudioDeviceObjects(javaVM, env, context);
     76   return 0;
     77 }
     78 
     79 void OpenSlesOutput::ClearAndroidAudioDeviceObjects() {
     80   AudioManagerJni::ClearAndroidAudioDeviceObjects();
     81 }
     82 
     83 int32_t OpenSlesOutput::Init() {
     84   assert(!initialized_);
     85 
     86   // Set up OpenSl engine.
     87   OPENSL_RETURN_ON_FAILURE(slCreateEngine(&sles_engine_, 1, kOption, 0,
     88                                           NULL, NULL),
     89                            -1);
     90   OPENSL_RETURN_ON_FAILURE((*sles_engine_)->Realize(sles_engine_,
     91                                                     SL_BOOLEAN_FALSE),
     92                            -1);
     93   OPENSL_RETURN_ON_FAILURE((*sles_engine_)->GetInterface(sles_engine_,
     94                                                          SL_IID_ENGINE,
     95                                                          &sles_engine_itf_),
     96                            -1);
     97   // Set up OpenSl output mix.
     98   OPENSL_RETURN_ON_FAILURE(
     99       (*sles_engine_itf_)->CreateOutputMix(sles_engine_itf_,
    100                                            &sles_output_mixer_,
    101                                            0,
    102                                            NULL,
    103                                            NULL),
    104       -1);
    105   OPENSL_RETURN_ON_FAILURE(
    106       (*sles_output_mixer_)->Realize(sles_output_mixer_,
    107                                      SL_BOOLEAN_FALSE),
    108       -1);
    109 
    110   if (!InitSampleRate()) {
    111     return -1;
    112   }
    113   AllocateBuffers();
    114   initialized_ = true;
    115   return 0;
    116 }
    117 
    118 int32_t OpenSlesOutput::Terminate() {
    119   // It is assumed that the caller has stopped recording before terminating.
    120   assert(!playing_);
    121   (*sles_output_mixer_)->Destroy(sles_output_mixer_);
    122   (*sles_engine_)->Destroy(sles_engine_);
    123   initialized_ = false;
    124   speaker_initialized_ = false;
    125   play_initialized_ = false;
    126   return 0;
    127 }
    128 
    129 int32_t OpenSlesOutput::PlayoutDeviceName(uint16_t index,
    130                                           char name[kAdmMaxDeviceNameSize],
    131                                           char guid[kAdmMaxGuidSize]) {
    132   assert(index == 0);
    133   // Empty strings.
    134   name[0] = '\0';
    135   guid[0] = '\0';
    136   return 0;
    137 }
    138 
    139 int32_t OpenSlesOutput::SetPlayoutDevice(uint16_t index) {
    140   assert(index == 0);
    141   return 0;
    142 }
    143 
    144 int32_t OpenSlesOutput::PlayoutIsAvailable(bool& available) {  // NOLINT
    145   available = true;
    146   return 0;
    147 }
    148 
    149 int32_t OpenSlesOutput::InitPlayout() {
    150   assert(initialized_);
    151   play_initialized_ = true;
    152   return 0;
    153 }
    154 
    155 int32_t OpenSlesOutput::StartPlayout() {
    156   assert(play_initialized_);
    157   assert(!playing_);
    158   if (!CreateAudioPlayer()) {
    159     return -1;
    160   }
    161 
    162   // Register callback to receive enqueued buffers.
    163   OPENSL_RETURN_ON_FAILURE(
    164       (*sles_player_sbq_itf_)->RegisterCallback(sles_player_sbq_itf_,
    165                                                 PlayerSimpleBufferQueueCallback,
    166                                                 this),
    167       -1);
    168   if (!EnqueueAllBuffers()) {
    169     return -1;
    170   }
    171 
    172   {
    173     // To prevent the compiler from e.g. optimizing the code to
    174     // playing_ = StartCbThreads() which wouldn't have been thread safe.
    175     CriticalSectionScoped lock(crit_sect_.get());
    176     playing_ = true;
    177   }
    178   if (!StartCbThreads()) {
    179     playing_ = false;
    180   }
    181   return 0;
    182 }
    183 
    184 int32_t OpenSlesOutput::StopPlayout() {
    185   StopCbThreads();
    186   DestroyAudioPlayer();
    187   playing_ = false;
    188   return 0;
    189 }
    190 
    191 int32_t OpenSlesOutput::InitSpeaker() {
    192   assert(!playing_);
    193   speaker_initialized_ = true;
    194   return 0;
    195 }
    196 
    197 int32_t OpenSlesOutput::SpeakerVolumeIsAvailable(bool& available) {  // NOLINT
    198   available = true;
    199   return 0;
    200 }
    201 
    202 int32_t OpenSlesOutput::SetSpeakerVolume(uint32_t volume) {
    203   assert(speaker_initialized_);
    204   assert(initialized_);
    205   // TODO(hellner): implement.
    206   return 0;
    207 }
    208 
    209 int32_t OpenSlesOutput::MaxSpeakerVolume(uint32_t& maxVolume) const {  // NOLINT
    210   assert(speaker_initialized_);
    211   assert(initialized_);
    212   // TODO(hellner): implement.
    213   maxVolume = 0;
    214   return 0;
    215 }
    216 
    217 int32_t OpenSlesOutput::MinSpeakerVolume(uint32_t& minVolume) const {  // NOLINT
    218   assert(speaker_initialized_);
    219   assert(initialized_);
    220   // TODO(hellner): implement.
    221   minVolume = 0;
    222   return 0;
    223 }
    224 
    225 int32_t OpenSlesOutput::SpeakerVolumeStepSize(
    226     uint16_t& stepSize) const {  // NOLINT
    227   assert(speaker_initialized_);
    228   stepSize = 1;
    229   return 0;
    230 }
    231 
    232 int32_t OpenSlesOutput::SpeakerMuteIsAvailable(bool& available) {  // NOLINT
    233   available = false;
    234   return 0;
    235 }
    236 
    237 int32_t OpenSlesOutput::StereoPlayoutIsAvailable(bool& available) {  // NOLINT
    238   available = false;
    239   return 0;
    240 }
    241 
    242 int32_t OpenSlesOutput::SetStereoPlayout(bool enable) {
    243   if (enable) {
    244     assert(false);
    245     return -1;
    246   }
    247   return 0;
    248 }
    249 
    250 int32_t OpenSlesOutput::StereoPlayout(bool& enabled) const {  // NOLINT
    251   enabled = kNumChannels == 2;
    252   return 0;
    253 }
    254 
    255 int32_t OpenSlesOutput::PlayoutBuffer(
    256     AudioDeviceModule::BufferType& type,  // NOLINT
    257     uint16_t& sizeMS) const {  // NOLINT
    258   type = AudioDeviceModule::kAdaptiveBufferSize;
    259   sizeMS = playout_delay_;
    260   return 0;
    261 }
    262 
    263 int32_t OpenSlesOutput::PlayoutDelay(uint16_t& delayMS) const {  // NOLINT
    264   delayMS = playout_delay_;
    265   return 0;
    266 }
    267 
    268 void OpenSlesOutput::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) {
    269   audio_buffer_ = audioBuffer;
    270 }
    271 
    272 int32_t OpenSlesOutput::SetLoudspeakerStatus(bool enable) {
    273   return 0;
    274 }
    275 
    276 int32_t OpenSlesOutput::GetLoudspeakerStatus(bool& enabled) const {  // NOLINT
    277   enabled = true;
    278   return 0;
    279 }
    280 
    281 int OpenSlesOutput::PlayoutDelayMs() {
    282   return playout_delay_;
    283 }
    284 
    285 bool OpenSlesOutput::InitSampleRate() {
    286   if (!SetLowLatency()) {
    287     speaker_sampling_rate_ = kDefaultSampleRate;
    288     // Default is to use 10ms buffers.
    289     buffer_size_samples_ = speaker_sampling_rate_ * 10 / 1000;
    290   }
    291   if (audio_buffer_->SetPlayoutSampleRate(speaker_sampling_rate_) < 0) {
    292     return false;
    293   }
    294   if (audio_buffer_->SetPlayoutChannels(kNumChannels) < 0) {
    295     return false;
    296   }
    297   UpdatePlayoutDelay();
    298   return true;
    299 }
    300 
    301 void OpenSlesOutput::UpdatePlayoutDelay() {
    302   // TODO(hellner): Add accurate delay estimate.
    303   // On average half the current buffer will have been played out.
    304   int outstanding_samples = (TotalBuffersUsed() - 0.5) * buffer_size_samples_;
    305   playout_delay_ = outstanding_samples / (speaker_sampling_rate_ / 1000);
    306 }
    307 
    308 bool OpenSlesOutput::SetLowLatency() {
    309   if (!audio_manager_.low_latency_supported()) {
    310     return false;
    311   }
    312   buffer_size_samples_ = audio_manager_.native_buffer_size();
    313   assert(buffer_size_samples_ > 0);
    314   speaker_sampling_rate_ = audio_manager_.native_output_sample_rate();
    315   assert(speaker_sampling_rate_ > 0);
    316   return true;
    317 }
    318 
    319 void OpenSlesOutput::CalculateNumFifoBuffersNeeded() {
    320   int number_of_bytes_needed =
    321       (speaker_sampling_rate_ * kNumChannels * sizeof(int16_t)) * 10 / 1000;
    322 
    323   // Ceiling of integer division: 1 + ((x - 1) / y)
    324   int buffers_per_10_ms =
    325       1 + ((number_of_bytes_needed - 1) / buffer_size_bytes_);
    326   // |num_fifo_buffers_needed_| is a multiple of 10ms of buffered up audio.
    327   num_fifo_buffers_needed_ = kNum10MsToBuffer * buffers_per_10_ms;
    328 }
    329 
    330 void OpenSlesOutput::AllocateBuffers() {
    331   // Allocate fine buffer to provide frames of the desired size.
    332   buffer_size_bytes_ = buffer_size_samples_ * kNumChannels * sizeof(int16_t);
    333   fine_buffer_.reset(new FineAudioBuffer(audio_buffer_, buffer_size_bytes_,
    334                                          speaker_sampling_rate_));
    335 
    336   // Allocate FIFO to handle passing buffers between processing and OpenSl
    337   // threads.
    338   CalculateNumFifoBuffersNeeded();  // Needs |buffer_size_bytes_| to be known
    339   assert(num_fifo_buffers_needed_ > 0);
    340   fifo_.reset(new SingleRwFifo(num_fifo_buffers_needed_));
    341 
    342   // Allocate the memory area to be used.
    343   play_buf_.reset(new scoped_ptr<int8_t[]>[TotalBuffersUsed()]);
    344   int required_buffer_size = fine_buffer_->RequiredBufferSizeBytes();
    345   for (int i = 0; i < TotalBuffersUsed(); ++i) {
    346     play_buf_[i].reset(new int8_t[required_buffer_size]);
    347   }
    348 }
    349 
    350 int OpenSlesOutput::TotalBuffersUsed() const {
    351   return num_fifo_buffers_needed_ + kNumOpenSlBuffers;
    352 }
    353 
    354 bool OpenSlesOutput::EnqueueAllBuffers() {
    355   active_queue_ = 0;
    356   number_underruns_ = 0;
    357   for (int i = 0; i < kNumOpenSlBuffers; ++i) {
    358     memset(play_buf_[i].get(), 0, buffer_size_bytes_);
    359     OPENSL_RETURN_ON_FAILURE(
    360         (*sles_player_sbq_itf_)->Enqueue(
    361             sles_player_sbq_itf_,
    362             reinterpret_cast<void*>(play_buf_[i].get()),
    363             buffer_size_bytes_),
    364         false);
    365   }
    366   // OpenSL playing has been stopped. I.e. only this thread is touching
    367   // |fifo_|.
    368   while (fifo_->size() != 0) {
    369     // Underrun might have happened when pushing new buffers to the FIFO.
    370     fifo_->Pop();
    371   }
    372   for (int i = kNumOpenSlBuffers; i < TotalBuffersUsed(); ++i) {
    373     memset(play_buf_[i].get(), 0, buffer_size_bytes_);
    374     fifo_->Push(play_buf_[i].get());
    375   }
    376   return true;
    377 }
    378 
    379 bool OpenSlesOutput::CreateAudioPlayer() {
    380   if (!event_.Start()) {
    381     assert(false);
    382     return false;
    383   }
    384   SLDataLocator_AndroidSimpleBufferQueue simple_buf_queue = {
    385     SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
    386     static_cast<SLuint32>(kNumOpenSlBuffers)
    387   };
    388   SLDataFormat_PCM configuration =
    389       webrtc_opensl::CreatePcmConfiguration(speaker_sampling_rate_);
    390   SLDataSource audio_source = { &simple_buf_queue, &configuration };
    391 
    392   SLDataLocator_OutputMix locator_outputmix;
    393   // Setup the data sink structure.
    394   locator_outputmix.locatorType = SL_DATALOCATOR_OUTPUTMIX;
    395   locator_outputmix.outputMix = sles_output_mixer_;
    396   SLDataSink audio_sink = { &locator_outputmix, NULL };
    397 
    398   // Interfaces for streaming audio data, setting volume and Android are needed.
    399   // Note the interfaces still need to be initialized. This only tells OpenSl
    400   // that the interfaces will be needed at some point.
    401   SLInterfaceID ids[kNumInterfaces] = {
    402     SL_IID_BUFFERQUEUE, SL_IID_VOLUME, SL_IID_ANDROIDCONFIGURATION };
    403   SLboolean req[kNumInterfaces] = {
    404     SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE };
    405   OPENSL_RETURN_ON_FAILURE(
    406       (*sles_engine_itf_)->CreateAudioPlayer(sles_engine_itf_, &sles_player_,
    407                                              &audio_source, &audio_sink,
    408                                              kNumInterfaces, ids, req),
    409       false);
    410   // Realize the player in synchronous mode.
    411   OPENSL_RETURN_ON_FAILURE((*sles_player_)->Realize(sles_player_,
    412                                                     SL_BOOLEAN_FALSE),
    413                            false);
    414   OPENSL_RETURN_ON_FAILURE(
    415       (*sles_player_)->GetInterface(sles_player_, SL_IID_PLAY,
    416                                     &sles_player_itf_),
    417       false);
    418   OPENSL_RETURN_ON_FAILURE(
    419       (*sles_player_)->GetInterface(sles_player_, SL_IID_BUFFERQUEUE,
    420                                     &sles_player_sbq_itf_),
    421       false);
    422   return true;
    423 }
    424 
    425 void OpenSlesOutput::DestroyAudioPlayer() {
    426   SLAndroidSimpleBufferQueueItf sles_player_sbq_itf = sles_player_sbq_itf_;
    427   {
    428     CriticalSectionScoped lock(crit_sect_.get());
    429     sles_player_sbq_itf_ = NULL;
    430     sles_player_itf_ = NULL;
    431   }
    432   event_.Stop();
    433   if (sles_player_sbq_itf) {
    434     // Release all buffers currently queued up.
    435     OPENSL_RETURN_ON_FAILURE(
    436         (*sles_player_sbq_itf)->Clear(sles_player_sbq_itf),
    437         VOID_RETURN);
    438   }
    439 
    440   if (sles_player_) {
    441     (*sles_player_)->Destroy(sles_player_);
    442     sles_player_ = NULL;
    443   }
    444 }
    445 
    446 bool OpenSlesOutput::HandleUnderrun(int event_id, int event_msg) {
    447   if (!playing_) {
    448     return false;
    449   }
    450   if (event_id == kNoUnderrun) {
    451     return false;
    452   }
    453   WEBRTC_TRACE(kTraceWarning, kTraceAudioDevice, id_, "Audio underrun");
    454   assert(event_id == kUnderrun);
    455   assert(event_msg > 0);
    456   // Wait for all enqueued buffers to be flushed.
    457   if (event_msg != kNumOpenSlBuffers) {
    458     return true;
    459   }
    460   // All buffers have been flushed. Restart the audio from scratch.
    461   // No need to check sles_player_itf_ as playing_ would be false before it is
    462   // set to NULL.
    463   OPENSL_RETURN_ON_FAILURE(
    464       (*sles_player_itf_)->SetPlayState(sles_player_itf_,
    465                                         SL_PLAYSTATE_STOPPED),
    466       true);
    467   EnqueueAllBuffers();
    468   OPENSL_RETURN_ON_FAILURE(
    469       (*sles_player_itf_)->SetPlayState(sles_player_itf_,
    470                                         SL_PLAYSTATE_PLAYING),
    471       true);
    472   return true;
    473 }
    474 
    475 void OpenSlesOutput::PlayerSimpleBufferQueueCallback(
    476     SLAndroidSimpleBufferQueueItf sles_player_sbq_itf,
    477     void* p_context) {
    478   OpenSlesOutput* audio_device = reinterpret_cast<OpenSlesOutput*>(p_context);
    479   audio_device->PlayerSimpleBufferQueueCallbackHandler(sles_player_sbq_itf);
    480 }
    481 
    482 void OpenSlesOutput::PlayerSimpleBufferQueueCallbackHandler(
    483     SLAndroidSimpleBufferQueueItf sles_player_sbq_itf) {
    484   if (fifo_->size() <= 0 || number_underruns_ > 0) {
    485     ++number_underruns_;
    486     event_.SignalEvent(kUnderrun, number_underruns_);
    487     return;
    488   }
    489   int8_t* audio = fifo_->Pop();
    490   if (audio)
    491   OPENSL_RETURN_ON_FAILURE(
    492       (*sles_player_sbq_itf)->Enqueue(sles_player_sbq_itf,
    493                                       audio,
    494                                       buffer_size_bytes_),
    495       VOID_RETURN);
    496   event_.SignalEvent(kNoUnderrun, 0);
    497 }
    498 
    499 bool OpenSlesOutput::StartCbThreads() {
    500   play_thread_.reset(ThreadWrapper::CreateThread(CbThread,
    501                                                  this,
    502                                                  kRealtimePriority,
    503                                                  "opensl_play_thread"));
    504   assert(play_thread_.get());
    505   OPENSL_RETURN_ON_FAILURE(
    506       (*sles_player_itf_)->SetPlayState(sles_player_itf_,
    507                                         SL_PLAYSTATE_PLAYING),
    508       false);
    509 
    510   unsigned int thread_id = 0;
    511   if (!play_thread_->Start(thread_id)) {
    512     assert(false);
    513     return false;
    514   }
    515   return true;
    516 }
    517 
    518 void OpenSlesOutput::StopCbThreads() {
    519   {
    520     CriticalSectionScoped lock(crit_sect_.get());
    521     playing_ = false;
    522   }
    523   if (sles_player_itf_) {
    524     OPENSL_RETURN_ON_FAILURE(
    525         (*sles_player_itf_)->SetPlayState(sles_player_itf_,
    526                                           SL_PLAYSTATE_STOPPED),
    527         VOID_RETURN);
    528   }
    529   if (play_thread_.get() == NULL) {
    530     return;
    531   }
    532   event_.Stop();
    533   if (play_thread_->Stop()) {
    534     play_thread_.reset();
    535   } else {
    536     assert(false);
    537   }
    538 }
    539 
    540 bool OpenSlesOutput::CbThread(void* context) {
    541   return reinterpret_cast<OpenSlesOutput*>(context)->CbThreadImpl();
    542 }
    543 
    544 bool OpenSlesOutput::CbThreadImpl() {
    545   assert(fine_buffer_.get() != NULL);
    546   int event_id;
    547   int event_msg;
    548   // event_ must not be waited on while a lock has been taken.
    549   event_.WaitOnEvent(&event_id, &event_msg);
    550 
    551   CriticalSectionScoped lock(crit_sect_.get());
    552   if (HandleUnderrun(event_id, event_msg)) {
    553     return playing_;
    554   }
    555   // if fifo_ is not full it means next item in memory must be free.
    556   while (fifo_->size() < num_fifo_buffers_needed_ && playing_) {
    557     int8_t* audio = play_buf_[active_queue_].get();
    558     fine_buffer_->GetBufferData(audio);
    559     fifo_->Push(audio);
    560     active_queue_ = (active_queue_ + 1) % TotalBuffersUsed();
    561   }
    562   return playing_;
    563 }
    564 
    565 }  // namespace webrtc
    566