Home | History | Annotate | Download | only in nuplayer 
      1 /*
      2  * Copyright (C) 2010 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 //#define LOG_NDEBUG 0
     18 #define LOG_TAG "NuPlayerRenderer"
     19 #include <utils/Log.h>
     20 
     21 #include "NuPlayerRenderer.h"
     22 #include <algorithm>
     23 #include <cutils/properties.h>
     24 #include <media/stagefright/foundation/ABuffer.h>
     25 #include <media/stagefright/foundation/ADebug.h>
     26 #include <media/stagefright/foundation/AMessage.h>
     27 #include <media/stagefright/foundation/AUtils.h>
     28 #include <media/stagefright/foundation/AWakeLock.h>
     29 #include <media/stagefright/MediaClock.h>
     30 #include <media/stagefright/MediaErrors.h>
     31 #include <media/stagefright/MetaData.h>
     32 #include <media/stagefright/Utils.h>
     33 #include <media/stagefright/VideoFrameScheduler.h>
     34 
     35 #include <inttypes.h>
     36 
     37 namespace android {
     38 
     39 /*
     40  * Example of common configuration settings in shell script form
     41 
     42    #Turn offload audio off (use PCM for Play Music) -- AudioPolicyManager
     43    adb shell setprop audio.offload.disable 1
     44 
     45    #Allow offload audio with video (requires offloading to be enabled) -- AudioPolicyManager
     46    adb shell setprop audio.offload.video 1
     47 
     48    #Use audio callbacks for PCM data
     49    adb shell setprop media.stagefright.audio.cbk 1
     50 
     51    #Use deep buffer for PCM data with video (it is generally enabled for audio-only)
     52    adb shell setprop media.stagefright.audio.deep 1
     53 
     54    #Set size of buffers for pcm audio sink in msec (example: 1000 msec)
     55    adb shell setprop media.stagefright.audio.sink 1000
     56 
     57  * These configurations take effect for the next track played (not the current track).
     58  */
     59 
     60 static inline bool getUseAudioCallbackSetting() {
     61     return property_get_bool("media.stagefright.audio.cbk", false /* default_value */);
     62 }
     63 
     64 static inline int32_t getAudioSinkPcmMsSetting() {
     65     return property_get_int32(
     66             "media.stagefright.audio.sink", 500 /* default_value */);
     67 }
     68 
     69 // Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink
     70 // is closed to allow the audio DSP to power down.
     71 static const int64_t kOffloadPauseMaxUs = 10000000ll;
     72 
     73 // Maximum allowed delay from AudioSink, 1.5 seconds.
     74 static const int64_t kMaxAllowedAudioSinkDelayUs = 1500000ll;
     75 
     76 static const int64_t kMinimumAudioClockUpdatePeriodUs = 20 /* msec */ * 1000;
     77 
     78 // static
     79 const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = {
     80         AUDIO_CHANNEL_NONE,
     81         AUDIO_OUTPUT_FLAG_NONE,
     82         AUDIO_FORMAT_INVALID,
     83         0, // mNumChannels
     84         0 // mSampleRate
     85 };
     86 
     87 // static
     88 const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
     89 
     90 NuPlayer::Renderer::Renderer(
     91         const sp<MediaPlayerBase::AudioSink> &sink,
     92         const sp<AMessage> &notify,
     93         uint32_t flags)
     94     : mAudioSink(sink),
     95       mUseVirtualAudioSink(false),
     96       mNotify(notify),
     97       mFlags(flags),
     98       mNumFramesWritten(0),
     99       mDrainAudioQueuePending(false),
    100       mDrainVideoQueuePending(false),
    101       mAudioQueueGeneration(0),
    102       mVideoQueueGeneration(0),
    103       mAudioDrainGeneration(0),
    104       mVideoDrainGeneration(0),
    105       mAudioEOSGeneration(0),
    106       mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT),
    107       mAudioFirstAnchorTimeMediaUs(-1),
    108       mAnchorTimeMediaUs(-1),
    109       mAnchorNumFramesWritten(-1),
    110       mVideoLateByUs(0ll),
    111       mHasAudio(false),
    112       mHasVideo(false),
    113       mNotifyCompleteAudio(false),
    114       mNotifyCompleteVideo(false),
    115       mSyncQueues(false),
    116       mPaused(false),
    117       mPauseDrainAudioAllowedUs(0),
    118       mVideoSampleReceived(false),
    119       mVideoRenderingStarted(false),
    120       mVideoRenderingStartGeneration(0),
    121       mAudioRenderingStartGeneration(0),
    122       mRenderingDataDelivered(false),
    123       mNextAudioClockUpdateTimeUs(-1),
    124       mLastAudioMediaTimeUs(-1),
    125       mAudioOffloadPauseTimeoutGeneration(0),
    126       mAudioTornDown(false),
    127       mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
    128       mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER),
    129       mTotalBuffersQueued(0),
    130       mLastAudioBufferDrained(0),
    131       mUseAudioCallback(false),
    132       mWakeLock(new AWakeLock()) {
    133     mMediaClock = new MediaClock;
    134     mPlaybackRate = mPlaybackSettings.mSpeed;
    135     mMediaClock->setPlaybackRate(mPlaybackRate);
    136 }
    137 
    138 NuPlayer::Renderer::~Renderer() {
    139     if (offloadingAudio()) {
    140         mAudioSink->stop();
    141         mAudioSink->flush();
    142         mAudioSink->close();
    143     }
    144 }
    145 
    146 void NuPlayer::Renderer::queueBuffer(
    147         bool audio,
    148         const sp<ABuffer> &buffer,
    149         const sp<AMessage> &notifyConsumed) {
    150     sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this);
    151     msg->setInt32("queueGeneration", getQueueGeneration(audio));
    152     msg->setInt32("audio", static_cast<int32_t>(audio));
    153     msg->setBuffer("buffer", buffer);
    154     msg->setMessage("notifyConsumed", notifyConsumed);
    155     msg->post();
    156 }
    157 
    158 void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
    159     CHECK_NE(finalResult, (status_t)OK);
    160 
    161     sp<AMessage> msg = new AMessage(kWhatQueueEOS, this);
    162     msg->setInt32("queueGeneration", getQueueGeneration(audio));
    163     msg->setInt32("audio", static_cast<int32_t>(audio));
    164     msg->setInt32("finalResult", finalResult);
    165     msg->post();
    166 }
    167 
    168 status_t NuPlayer::Renderer::setPlaybackSettings(const AudioPlaybackRate &rate) {
    169     sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this);
    170     writeToAMessage(msg, rate);
    171     sp<AMessage> response;
    172     status_t err = msg->postAndAwaitResponse(&response);
    173     if (err == OK && response != NULL) {
    174         CHECK(response->findInt32("err", &err));
    175     }
    176     return err;
    177 }
    178 
    179 status_t NuPlayer::Renderer::onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */) {
    180     if (rate.mSpeed == 0.f) {
    181         onPause();
    182         // don't call audiosink's setPlaybackRate if pausing, as pitch does not
    183         // have to correspond to the any non-0 speed (e.g old speed). Keep
    184         // settings nonetheless, using the old speed, in case audiosink changes.
    185         AudioPlaybackRate newRate = rate;
    186         newRate.mSpeed = mPlaybackSettings.mSpeed;
    187         mPlaybackSettings = newRate;
    188         return OK;
    189     }
    190 
    191     if (mAudioSink != NULL && mAudioSink->ready()) {
    192         status_t err = mAudioSink->setPlaybackRate(rate);
    193         if (err != OK) {
    194             return err;
    195         }
    196     }
    197     mPlaybackSettings = rate;
    198     mPlaybackRate = rate.mSpeed;
    199     mMediaClock->setPlaybackRate(mPlaybackRate);
    200     return OK;
    201 }
    202 
    203 status_t NuPlayer::Renderer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
    204     sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this);
    205     sp<AMessage> response;
    206     status_t err = msg->postAndAwaitResponse(&response);
    207     if (err == OK && response != NULL) {
    208         CHECK(response->findInt32("err", &err));
    209         if (err == OK) {
    210             readFromAMessage(response, rate);
    211         }
    212     }
    213     return err;
    214 }
    215 
    216 status_t NuPlayer::Renderer::onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
    217     if (mAudioSink != NULL && mAudioSink->ready()) {
    218         status_t err = mAudioSink->getPlaybackRate(rate);
    219         if (err == OK) {
    220             if (!isAudioPlaybackRateEqual(*rate, mPlaybackSettings)) {
    221                 ALOGW("correcting mismatch in internal/external playback rate");
    222             }
    223             // get playback settings used by audiosink, as it may be
    224             // slightly off due to audiosink not taking small changes.
    225             mPlaybackSettings = *rate;
    226             if (mPaused) {
    227                 rate->mSpeed = 0.f;
    228             }
    229         }
    230         return err;
    231     }
    232     *rate = mPlaybackSettings;
    233     return OK;
    234 }
    235 
    236 status_t NuPlayer::Renderer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) {
    237     sp<AMessage> msg = new AMessage(kWhatConfigSync, this);
    238     writeToAMessage(msg, sync, videoFpsHint);
    239     sp<AMessage> response;
    240     status_t err = msg->postAndAwaitResponse(&response);
    241     if (err == OK && response != NULL) {
    242         CHECK(response->findInt32("err", &err));
    243     }
    244     return err;
    245 }
    246 
    247 status_t NuPlayer::Renderer::onConfigSync(const AVSyncSettings &sync, float videoFpsHint __unused) {
    248     if (sync.mSource != AVSYNC_SOURCE_DEFAULT) {
    249         return BAD_VALUE;
    250     }
    251     // TODO: support sync sources
    252     return INVALID_OPERATION;
    253 }
    254 
    255 status_t NuPlayer::Renderer::getSyncSettings(AVSyncSettings *sync, float *videoFps) {
    256     sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this);
    257     sp<AMessage> response;
    258     status_t err = msg->postAndAwaitResponse(&response);
    259     if (err == OK && response != NULL) {
    260         CHECK(response->findInt32("err", &err));
    261         if (err == OK) {
    262             readFromAMessage(response, sync, videoFps);
    263         }
    264     }
    265     return err;
    266 }
    267 
    268 status_t NuPlayer::Renderer::onGetSyncSettings(
    269         AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) {
    270     *sync = mSyncSettings;
    271     *videoFps = -1.f;
    272     return OK;
    273 }
    274 
    275 void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) {
    276     {
    277         Mutex::Autolock autoLock(mLock);
    278         if (audio) {
    279             mNotifyCompleteAudio |= notifyComplete;
    280             clearAudioFirstAnchorTime_l();
    281             ++mAudioQueueGeneration;
    282             ++mAudioDrainGeneration;
    283         } else {
    284             mNotifyCompleteVideo |= notifyComplete;
    285             ++mVideoQueueGeneration;
    286             ++mVideoDrainGeneration;
    287         }
    288 
    289         clearAnchorTime_l();
    290         mVideoLateByUs = 0;
    291         mSyncQueues = false;
    292     }
    293 
    294     sp<AMessage> msg = new AMessage(kWhatFlush, this);
    295     msg->setInt32("audio", static_cast<int32_t>(audio));
    296     msg->post();
    297 }
    298 
    299 void NuPlayer::Renderer::signalTimeDiscontinuity() {
    300 }
    301 
    302 void NuPlayer::Renderer::signalDisableOffloadAudio() {
    303     (new AMessage(kWhatDisableOffloadAudio, this))->post();
    304 }
    305 
    306 void NuPlayer::Renderer::signalEnableOffloadAudio() {
    307     (new AMessage(kWhatEnableOffloadAudio, this))->post();
    308 }
    309 
    310 void NuPlayer::Renderer::pause() {
    311     (new AMessage(kWhatPause, this))->post();
    312 }
    313 
    314 void NuPlayer::Renderer::resume() {
    315     (new AMessage(kWhatResume, this))->post();
    316 }
    317 
    318 void NuPlayer::Renderer::setVideoFrameRate(float fps) {
    319     sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this);
    320     msg->setFloat("frame-rate", fps);
    321     msg->post();
    322 }
    323 
    324 // Called on any threads without mLock acquired.
    325 status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) {
    326     status_t result = mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
    327     if (result == OK) {
    328         return result;
    329     }
    330 
    331     // MediaClock has not started yet. Try to start it if possible.
    332     {
    333         Mutex::Autolock autoLock(mLock);
    334         if (mAudioFirstAnchorTimeMediaUs == -1) {
    335             return result;
    336         }
    337 
    338         AudioTimestamp ts;
    339         status_t res = mAudioSink->getTimestamp(ts);
    340         if (res != OK) {
    341             return result;
    342         }
    343 
    344         // AudioSink has rendered some frames.
    345         int64_t nowUs = ALooper::GetNowUs();
    346         int64_t nowMediaUs = mAudioSink->getPlayedOutDurationUs(nowUs)
    347                 + mAudioFirstAnchorTimeMediaUs;
    348         mMediaClock->updateAnchor(nowMediaUs, nowUs, -1);
    349     }
    350 
    351     return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
    352 }
    353 
    354 void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() {
    355     mAudioFirstAnchorTimeMediaUs = -1;
    356     mMediaClock->setStartingTimeMedia(-1);
    357 }
    358 
    359 void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) {
    360     if (mAudioFirstAnchorTimeMediaUs == -1) {
    361         mAudioFirstAnchorTimeMediaUs = mediaUs;
    362         mMediaClock->setStartingTimeMedia(mediaUs);
    363     }
    364 }
    365 
    366 void NuPlayer::Renderer::clearAnchorTime_l() {
    367     mMediaClock->clearAnchor();
    368     mAnchorTimeMediaUs = -1;
    369     mAnchorNumFramesWritten = -1;
    370 }
    371 
    372 void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) {
    373     Mutex::Autolock autoLock(mLock);
    374     mVideoLateByUs = lateUs;
    375 }
    376 
    377 int64_t NuPlayer::Renderer::getVideoLateByUs() {
    378     Mutex::Autolock autoLock(mLock);
    379     return mVideoLateByUs;
    380 }
    381 
    382 status_t NuPlayer::Renderer::openAudioSink(
    383         const sp<AMessage> &format,
    384         bool offloadOnly,
    385         bool hasVideo,
    386         uint32_t flags,
    387         bool *isOffloaded) {
    388     sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this);
    389     msg->setMessage("format", format);
    390     msg->setInt32("offload-only", offloadOnly);
    391     msg->setInt32("has-video", hasVideo);
    392     msg->setInt32("flags", flags);
    393 
    394     sp<AMessage> response;
    395     msg->postAndAwaitResponse(&response);
    396 
    397     int32_t err;
    398     if (!response->findInt32("err", &err)) {
    399         err = INVALID_OPERATION;
    400     } else if (err == OK && isOffloaded != NULL) {
    401         int32_t offload;
    402         CHECK(response->findInt32("offload", &offload));
    403         *isOffloaded = (offload != 0);
    404     }
    405     return err;
    406 }
    407 
    408 void NuPlayer::Renderer::closeAudioSink() {
    409     sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this);
    410 
    411     sp<AMessage> response;
    412     msg->postAndAwaitResponse(&response);
    413 }
    414 
    415 void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
    416     switch (msg->what()) {
    417         case kWhatOpenAudioSink:
    418         {
    419             sp<AMessage> format;
    420             CHECK(msg->findMessage("format", &format));
    421 
    422             int32_t offloadOnly;
    423             CHECK(msg->findInt32("offload-only", &offloadOnly));
    424 
    425             int32_t hasVideo;
    426             CHECK(msg->findInt32("has-video", &hasVideo));
    427 
    428             uint32_t flags;
    429             CHECK(msg->findInt32("flags", (int32_t *)&flags));
    430 
    431             status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags);
    432 
    433             sp<AMessage> response = new AMessage;
    434             response->setInt32("err", err);
    435             response->setInt32("offload", offloadingAudio());
    436 
    437             sp<AReplyToken> replyID;
    438             CHECK(msg->senderAwaitsResponse(&replyID));
    439             response->postReply(replyID);
    440 
    441             break;
    442         }
    443 
    444         case kWhatCloseAudioSink:
    445         {
    446             sp<AReplyToken> replyID;
    447             CHECK(msg->senderAwaitsResponse(&replyID));
    448 
    449             onCloseAudioSink();
    450 
    451             sp<AMessage> response = new AMessage;
    452             response->postReply(replyID);
    453             break;
    454         }
    455 
    456         case kWhatStopAudioSink:
    457         {
    458             mAudioSink->stop();
    459             break;
    460         }
    461 
    462         case kWhatDrainAudioQueue:
    463         {
    464             mDrainAudioQueuePending = false;
    465 
    466             int32_t generation;
    467             CHECK(msg->findInt32("drainGeneration", &generation));
    468             if (generation != getDrainGeneration(true /* audio */)) {
    469                 break;
    470             }
    471 
    472             if (onDrainAudioQueue()) {
    473                 uint32_t numFramesPlayed;
    474                 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
    475                          (status_t)OK);
    476 
    477                 uint32_t numFramesPendingPlayout =
    478                     mNumFramesWritten - numFramesPlayed;
    479 
    480                 // This is how long the audio sink will have data to
    481                 // play back.
    482                 int64_t delayUs =
    483                     mAudioSink->msecsPerFrame()
    484                         * numFramesPendingPlayout * 1000ll;
    485                 if (mPlaybackRate > 1.0f) {
    486                     delayUs /= mPlaybackRate;
    487                 }
    488 
    489                 // Let's give it more data after about half that time
    490                 // has elapsed.
    491                 delayUs /= 2;
    492                 // check the buffer size to estimate maximum delay permitted.
    493                 const int64_t maxDrainDelayUs = std::max(
    494                         mAudioSink->getBufferDurationInUs(), (int64_t)500000 /* half second */);
    495                 ALOGD_IF(delayUs > maxDrainDelayUs, "postDrainAudioQueue long delay: %lld > %lld",
    496                         (long long)delayUs, (long long)maxDrainDelayUs);
    497                 Mutex::Autolock autoLock(mLock);
    498                 postDrainAudioQueue_l(delayUs);
    499             }
    500             break;
    501         }
    502 
    503         case kWhatDrainVideoQueue:
    504         {
    505             int32_t generation;
    506             CHECK(msg->findInt32("drainGeneration", &generation));
    507             if (generation != getDrainGeneration(false /* audio */)) {
    508                 break;
    509             }
    510 
    511             mDrainVideoQueuePending = false;
    512 
    513             onDrainVideoQueue();
    514 
    515             postDrainVideoQueue();
    516             break;
    517         }
    518 
    519         case kWhatPostDrainVideoQueue:
    520         {
    521             int32_t generation;
    522             CHECK(msg->findInt32("drainGeneration", &generation));
    523             if (generation != getDrainGeneration(false /* audio */)) {
    524                 break;
    525             }
    526 
    527             mDrainVideoQueuePending = false;
    528             postDrainVideoQueue();
    529             break;
    530         }
    531 
    532         case kWhatQueueBuffer:
    533         {
    534             onQueueBuffer(msg);
    535             break;
    536         }
    537 
    538         case kWhatQueueEOS:
    539         {
    540             onQueueEOS(msg);
    541             break;
    542         }
    543 
    544         case kWhatEOS:
    545         {
    546             int32_t generation;
    547             CHECK(msg->findInt32("audioEOSGeneration", &generation));
    548             if (generation != mAudioEOSGeneration) {
    549                 break;
    550             }
    551             status_t finalResult;
    552             CHECK(msg->findInt32("finalResult", &finalResult));
    553             notifyEOS(true /* audio */, finalResult);
    554             break;
    555         }
    556 
    557         case kWhatConfigPlayback:
    558         {
    559             sp<AReplyToken> replyID;
    560             CHECK(msg->senderAwaitsResponse(&replyID));
    561             AudioPlaybackRate rate;
    562             readFromAMessage(msg, &rate);
    563             status_t err = onConfigPlayback(rate);
    564             sp<AMessage> response = new AMessage;
    565             response->setInt32("err", err);
    566             response->postReply(replyID);
    567             break;
    568         }
    569 
    570         case kWhatGetPlaybackSettings:
    571         {
    572             sp<AReplyToken> replyID;
    573             CHECK(msg->senderAwaitsResponse(&replyID));
    574             AudioPlaybackRate rate = AUDIO_PLAYBACK_RATE_DEFAULT;
    575             status_t err = onGetPlaybackSettings(&rate);
    576             sp<AMessage> response = new AMessage;
    577             if (err == OK) {
    578                 writeToAMessage(response, rate);
    579             }
    580             response->setInt32("err", err);
    581             response->postReply(replyID);
    582             break;
    583         }
    584 
    585         case kWhatConfigSync:
    586         {
    587             sp<AReplyToken> replyID;
    588             CHECK(msg->senderAwaitsResponse(&replyID));
    589             AVSyncSettings sync;
    590             float videoFpsHint;
    591             readFromAMessage(msg, &sync, &videoFpsHint);
    592             status_t err = onConfigSync(sync, videoFpsHint);
    593             sp<AMessage> response = new AMessage;
    594             response->setInt32("err", err);
    595             response->postReply(replyID);
    596             break;
    597         }
    598 
    599         case kWhatGetSyncSettings:
    600         {
    601             sp<AReplyToken> replyID;
    602             CHECK(msg->senderAwaitsResponse(&replyID));
    603 
    604             ALOGV("kWhatGetSyncSettings");
    605             AVSyncSettings sync;
    606             float videoFps = -1.f;
    607             status_t err = onGetSyncSettings(&sync, &videoFps);
    608             sp<AMessage> response = new AMessage;
    609             if (err == OK) {
    610                 writeToAMessage(response, sync, videoFps);
    611             }
    612             response->setInt32("err", err);
    613             response->postReply(replyID);
    614             break;
    615         }
    616 
    617         case kWhatFlush:
    618         {
    619             onFlush(msg);
    620             break;
    621         }
    622 
    623         case kWhatDisableOffloadAudio:
    624         {
    625             onDisableOffloadAudio();
    626             break;
    627         }
    628 
    629         case kWhatEnableOffloadAudio:
    630         {
    631             onEnableOffloadAudio();
    632             break;
    633         }
    634 
    635         case kWhatPause:
    636         {
    637             onPause();
    638             break;
    639         }
    640 
    641         case kWhatResume:
    642         {
    643             onResume();
    644             break;
    645         }
    646 
    647         case kWhatSetVideoFrameRate:
    648         {
    649             float fps;
    650             CHECK(msg->findFloat("frame-rate", &fps));
    651             onSetVideoFrameRate(fps);
    652             break;
    653         }
    654 
    655         case kWhatAudioTearDown:
    656         {
    657             int32_t reason;
    658             CHECK(msg->findInt32("reason", &reason));
    659 
    660             onAudioTearDown((AudioTearDownReason)reason);
    661             break;
    662         }
    663 
    664         case kWhatAudioOffloadPauseTimeout:
    665         {
    666             int32_t generation;
    667             CHECK(msg->findInt32("drainGeneration", &generation));
    668             if (generation != mAudioOffloadPauseTimeoutGeneration) {
    669                 break;
    670             }
    671             ALOGV("Audio Offload tear down due to pause timeout.");
    672             onAudioTearDown(kDueToTimeout);
    673             mWakeLock->release();
    674             break;
    675         }
    676 
    677         default:
    678             TRESPASS();
    679             break;
    680     }
    681 }
    682 
    683 void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
    684     if (mDrainAudioQueuePending || mSyncQueues || mUseAudioCallback) {
    685         return;
    686     }
    687 
    688     if (mAudioQueue.empty()) {
    689         return;
    690     }
    691 
    692     // FIXME: if paused, wait until AudioTrack stop() is complete before delivering data.
    693     if (mPaused) {
    694         const int64_t diffUs = mPauseDrainAudioAllowedUs - ALooper::GetNowUs();
    695         if (diffUs > delayUs) {
    696             delayUs = diffUs;
    697         }
    698     }
    699 
    700     mDrainAudioQueuePending = true;
    701     sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this);
    702     msg->setInt32("drainGeneration", mAudioDrainGeneration);
    703     msg->post(delayUs);
    704 }
    705 
    706 void NuPlayer::Renderer::prepareForMediaRenderingStart_l() {
    707     mAudioRenderingStartGeneration = mAudioDrainGeneration;
    708     mVideoRenderingStartGeneration = mVideoDrainGeneration;
    709     mRenderingDataDelivered = false;
    710 }
    711 
    712 void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() {
    713     if (mVideoRenderingStartGeneration == mVideoDrainGeneration &&
    714         mAudioRenderingStartGeneration == mAudioDrainGeneration) {
    715         mRenderingDataDelivered = true;
    716         if (mPaused) {
    717             return;
    718         }
    719         mVideoRenderingStartGeneration = -1;
    720         mAudioRenderingStartGeneration = -1;
    721 
    722         sp<AMessage> notify = mNotify->dup();
    723         notify->setInt32("what", kWhatMediaRenderingStart);
    724         notify->post();
    725     }
    726 }
    727 
    728 // static
    729 size_t NuPlayer::Renderer::AudioSinkCallback(
    730         MediaPlayerBase::AudioSink * /* audioSink */,
    731         void *buffer,
    732         size_t size,
    733         void *cookie,
    734         MediaPlayerBase::AudioSink::cb_event_t event) {
    735     NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
    736 
    737     switch (event) {
    738         case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
    739         {
    740             return me->fillAudioBuffer(buffer, size);
    741             break;
    742         }
    743 
    744         case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
    745         {
    746             ALOGV("AudioSink::CB_EVENT_STREAM_END");
    747             me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM);
    748             break;
    749         }
    750 
    751         case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
    752         {
    753             ALOGV("AudioSink::CB_EVENT_TEAR_DOWN");
    754             me->notifyAudioTearDown(kDueToError);
    755             break;
    756         }
    757     }
    758 
    759     return 0;
    760 }
    761 
    762 size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
    763     Mutex::Autolock autoLock(mLock);
    764 
    765     if (!mUseAudioCallback) {
    766         return 0;
    767     }
    768 
    769     bool hasEOS = false;
    770 
    771     size_t sizeCopied = 0;
    772     bool firstEntry = true;
    773     QueueEntry *entry;  // will be valid after while loop if hasEOS is set.
    774     while (sizeCopied < size && !mAudioQueue.empty()) {
    775         entry = &*mAudioQueue.begin();
    776 
    777         if (entry->mBuffer == NULL) { // EOS
    778             hasEOS = true;
    779             mAudioQueue.erase(mAudioQueue.begin());
    780             break;
    781         }
    782 
    783         if (firstEntry && entry->mOffset == 0) {
    784             firstEntry = false;
    785             int64_t mediaTimeUs;
    786             CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
    787             ALOGV("fillAudioBuffer: rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
    788             setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
    789         }
    790 
    791         size_t copy = entry->mBuffer->size() - entry->mOffset;
    792         size_t sizeRemaining = size - sizeCopied;
    793         if (copy > sizeRemaining) {
    794             copy = sizeRemaining;
    795         }
    796 
    797         memcpy((char *)buffer + sizeCopied,
    798                entry->mBuffer->data() + entry->mOffset,
    799                copy);
    800 
    801         entry->mOffset += copy;
    802         if (entry->mOffset == entry->mBuffer->size()) {
    803             entry->mNotifyConsumed->post();
    804             mAudioQueue.erase(mAudioQueue.begin());
    805             entry = NULL;
    806         }
    807         sizeCopied += copy;
    808 
    809         notifyIfMediaRenderingStarted_l();
    810     }
    811 
    812     if (mAudioFirstAnchorTimeMediaUs >= 0) {
    813         int64_t nowUs = ALooper::GetNowUs();
    814         int64_t nowMediaUs =
    815             mAudioFirstAnchorTimeMediaUs + mAudioSink->getPlayedOutDurationUs(nowUs);
    816         // we don't know how much data we are queueing for offloaded tracks.
    817         mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX);
    818     }
    819 
    820     // for non-offloaded audio, we need to compute the frames written because
    821     // there is no EVENT_STREAM_END notification. The frames written gives
    822     // an estimate on the pending played out duration.
    823     if (!offloadingAudio()) {
    824         mNumFramesWritten += sizeCopied / mAudioSink->frameSize();
    825     }
    826 
    827     if (hasEOS) {
    828         (new AMessage(kWhatStopAudioSink, this))->post();
    829         // As there is currently no EVENT_STREAM_END callback notification for
    830         // non-offloaded audio tracks, we need to post the EOS ourselves.
    831         if (!offloadingAudio()) {
    832             int64_t postEOSDelayUs = 0;
    833             if (mAudioSink->needsTrailingPadding()) {
    834                 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
    835             }
    836             ALOGV("fillAudioBuffer: notifyEOS "
    837                     "mNumFramesWritten:%u  finalResult:%d  postEOSDelay:%lld",
    838                     mNumFramesWritten, entry->mFinalResult, (long long)postEOSDelayUs);
    839             notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
    840         }
    841     }
    842     return sizeCopied;
    843 }
    844 
    845 void NuPlayer::Renderer::drainAudioQueueUntilLastEOS() {
    846     List<QueueEntry>::iterator it = mAudioQueue.begin(), itEOS = it;
    847     bool foundEOS = false;
    848     while (it != mAudioQueue.end()) {
    849         int32_t eos;
    850         QueueEntry *entry = &*it++;
    851         if (entry->mBuffer == NULL
    852                 || (entry->mNotifyConsumed->findInt32("eos", &eos) && eos != 0)) {
    853             itEOS = it;
    854             foundEOS = true;
    855         }
    856     }
    857 
    858     if (foundEOS) {
    859         // post all replies before EOS and drop the samples
    860         for (it = mAudioQueue.begin(); it != itEOS; it++) {
    861             if (it->mBuffer == NULL) {
    862                 // delay doesn't matter as we don't even have an AudioTrack
    863                 notifyEOS(true /* audio */, it->mFinalResult);
    864             } else {
    865                 it->mNotifyConsumed->post();
    866             }
    867         }
    868         mAudioQueue.erase(mAudioQueue.begin(), itEOS);
    869     }
    870 }
    871 
    872 bool NuPlayer::Renderer::onDrainAudioQueue() {
    873     // do not drain audio during teardown as queued buffers may be invalid.
    874     if (mAudioTornDown) {
    875         return false;
    876     }
    877     // TODO: This call to getPosition checks if AudioTrack has been created
    878     // in AudioSink before draining audio. If AudioTrack doesn't exist, then
    879     // CHECKs on getPosition will fail.
    880     // We still need to figure out why AudioTrack is not created when
    881     // this function is called. One possible reason could be leftover
    882     // audio. Another possible place is to check whether decoder
    883     // has received INFO_FORMAT_CHANGED as the first buffer since
    884     // AudioSink is opened there, and possible interactions with flush
    885     // immediately after start. Investigate error message
    886     // "vorbis_dsp_synthesis returned -135", along with RTSP.
    887     uint32_t numFramesPlayed;
    888     if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
    889         // When getPosition fails, renderer will not reschedule the draining
    890         // unless new samples are queued.
    891         // If we have pending EOS (or "eos" marker for discontinuities), we need
    892         // to post these now as NuPlayerDecoder might be waiting for it.
    893         drainAudioQueueUntilLastEOS();
    894 
    895         ALOGW("onDrainAudioQueue(): audio sink is not ready");
    896         return false;
    897     }
    898 
    899 #if 0
    900     ssize_t numFramesAvailableToWrite =
    901         mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
    902 
    903     if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
    904         ALOGI("audio sink underrun");
    905     } else {
    906         ALOGV("audio queue has %d frames left to play",
    907              mAudioSink->frameCount() - numFramesAvailableToWrite);
    908     }
    909 #endif
    910 
    911     uint32_t prevFramesWritten = mNumFramesWritten;
    912     while (!mAudioQueue.empty()) {
    913         QueueEntry *entry = &*mAudioQueue.begin();
    914 
    915         mLastAudioBufferDrained = entry->mBufferOrdinal;
    916 
    917         if (entry->mBuffer == NULL) {
    918             // EOS
    919             int64_t postEOSDelayUs = 0;
    920             if (mAudioSink->needsTrailingPadding()) {
    921                 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
    922             }
    923             notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
    924             mLastAudioMediaTimeUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
    925 
    926             mAudioQueue.erase(mAudioQueue.begin());
    927             entry = NULL;
    928             if (mAudioSink->needsTrailingPadding()) {
    929                 // If we're not in gapless playback (i.e. through setNextPlayer), we
    930                 // need to stop the track here, because that will play out the last
    931                 // little bit at the end of the file. Otherwise short files won't play.
    932                 mAudioSink->stop();
    933                 mNumFramesWritten = 0;
    934             }
    935             return false;
    936         }
    937 
    938         // ignore 0-sized buffer which could be EOS marker with no data
    939         if (entry->mOffset == 0 && entry->mBuffer->size() > 0) {
    940             int64_t mediaTimeUs;
    941             CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
    942             ALOGV("onDrainAudioQueue: rendering audio at media time %.2f secs",
    943                     mediaTimeUs / 1E6);
    944             onNewAudioMediaTime(mediaTimeUs);
    945         }
    946 
    947         size_t copy = entry->mBuffer->size() - entry->mOffset;
    948 
    949         ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset,
    950                                             copy, false /* blocking */);
    951         if (written < 0) {
    952             // An error in AudioSink write. Perhaps the AudioSink was not properly opened.
    953             if (written == WOULD_BLOCK) {
    954                 ALOGV("AudioSink write would block when writing %zu bytes", copy);
    955             } else {
    956                 ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy);
    957                 // This can only happen when AudioSink was opened with doNotReconnect flag set to
    958                 // true, in which case the NuPlayer will handle the reconnect.
    959                 notifyAudioTearDown(kDueToError);
    960             }
    961             break;
    962         }
    963 
    964         entry->mOffset += written;
    965         size_t remainder = entry->mBuffer->size() - entry->mOffset;
    966         if ((ssize_t)remainder < mAudioSink->frameSize()) {
    967             if (remainder > 0) {
    968                 ALOGW("Corrupted audio buffer has fractional frames, discarding %zu bytes.",
    969                         remainder);
    970                 entry->mOffset += remainder;
    971                 copy -= remainder;
    972             }
    973 
    974             entry->mNotifyConsumed->post();
    975             mAudioQueue.erase(mAudioQueue.begin());
    976 
    977             entry = NULL;
    978         }
    979 
    980         size_t copiedFrames = written / mAudioSink->frameSize();
    981         mNumFramesWritten += copiedFrames;
    982 
    983         {
    984             Mutex::Autolock autoLock(mLock);
    985             int64_t maxTimeMedia;
    986             maxTimeMedia =
    987                 mAnchorTimeMediaUs +
    988                         (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL)
    989                                 * 1000LL * mAudioSink->msecsPerFrame());
    990             mMediaClock->updateMaxTimeMedia(maxTimeMedia);
    991 
    992             notifyIfMediaRenderingStarted_l();
    993         }
    994 
    995         if (written != (ssize_t)copy) {
    996             // A short count was received from AudioSink::write()
    997             //
    998             // AudioSink write is called in non-blocking mode.
    999             // It may return with a short count when:
   1000             //
   1001             // 1) Size to be copied is not a multiple of the frame size. Fractional frames are
   1002             //    discarded.
   1003             // 2) The data to be copied exceeds the available buffer in AudioSink.
   1004             // 3) An error occurs and data has been partially copied to the buffer in AudioSink.
   1005             // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
   1006 
   1007             // (Case 1)
   1008             // Must be a multiple of the frame size.  If it is not a multiple of a frame size, it
   1009             // needs to fail, as we should not carry over fractional frames between calls.
   1010             CHECK_EQ(copy % mAudioSink->frameSize(), 0);
   1011 
   1012             // (Case 2, 3, 4)
   1013             // Return early to the caller.
   1014             // Beware of calling immediately again as this may busy-loop if you are not careful.
   1015             ALOGV("AudioSink write short frame count %zd < %zu", written, copy);
   1016             break;
   1017         }
   1018     }
   1019 
   1020     // calculate whether we need to reschedule another write.
   1021     bool reschedule = !mAudioQueue.empty()
   1022             && (!mPaused
   1023                 || prevFramesWritten != mNumFramesWritten); // permit pause to fill buffers
   1024     //ALOGD("reschedule:%d  empty:%d  mPaused:%d  prevFramesWritten:%u  mNumFramesWritten:%u",
   1025     //        reschedule, mAudioQueue.empty(), mPaused, prevFramesWritten, mNumFramesWritten);
   1026     return reschedule;
   1027 }
   1028 
   1029 int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) {
   1030     int32_t sampleRate = offloadingAudio() ?
   1031             mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate;
   1032     if (sampleRate == 0) {
   1033         ALOGE("sampleRate is 0 in %s mode", offloadingAudio() ? "offload" : "non-offload");
   1034         return 0;
   1035     }
   1036     // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours.
   1037     return (int64_t)((int32_t)numFrames * 1000000LL / sampleRate);
   1038 }
   1039 
   1040 // Calculate duration of pending samples if played at normal rate (i.e., 1.0).
   1041 int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) {
   1042     int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
   1043     if (mUseVirtualAudioSink) {
   1044         int64_t nowUs = ALooper::GetNowUs();
   1045         int64_t mediaUs;
   1046         if (mMediaClock->getMediaTime(nowUs, &mediaUs) != OK) {
   1047             return 0ll;
   1048         } else {
   1049             return writtenAudioDurationUs - (mediaUs - mAudioFirstAnchorTimeMediaUs);
   1050         }
   1051     }
   1052     return writtenAudioDurationUs - mAudioSink->getPlayedOutDurationUs(nowUs);
   1053 }
   1054 
   1055 int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) {
   1056     int64_t realUs;
   1057     if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
   1058         // If failed to get current position, e.g. due to audio clock is
   1059         // not ready, then just play out video immediately without delay.
   1060         return nowUs;
   1061     }
   1062     return realUs;
   1063 }
   1064 
   1065 void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) {
   1066     Mutex::Autolock autoLock(mLock);
   1067     // TRICKY: vorbis decoder generates multiple frames with the same
   1068     // timestamp, so only update on the first frame with a given timestamp
   1069     if (mediaTimeUs == mAnchorTimeMediaUs) {
   1070         return;
   1071     }
   1072     setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
   1073 
   1074     // mNextAudioClockUpdateTimeUs is -1 if we're waiting for audio sink to start
   1075     if (mNextAudioClockUpdateTimeUs == -1) {
   1076         AudioTimestamp ts;
   1077         if (mAudioSink->getTimestamp(ts) == OK && ts.mPosition > 0) {
   1078             mNextAudioClockUpdateTimeUs = 0; // start our clock updates
   1079         }
   1080     }
   1081     int64_t nowUs = ALooper::GetNowUs();
   1082     if (mNextAudioClockUpdateTimeUs >= 0) {
   1083         if (nowUs >= mNextAudioClockUpdateTimeUs) {
   1084             int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs);
   1085             mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs);
   1086             mUseVirtualAudioSink = false;
   1087             mNextAudioClockUpdateTimeUs = nowUs + kMinimumAudioClockUpdatePeriodUs;
   1088         }
   1089     } else {
   1090         int64_t unused;
   1091         if ((mMediaClock->getMediaTime(nowUs, &unused) != OK)
   1092                 && (getDurationUsIfPlayedAtSampleRate(mNumFramesWritten)
   1093                         > kMaxAllowedAudioSinkDelayUs)) {
   1094             // Enough data has been sent to AudioSink, but AudioSink has not rendered
   1095             // any data yet. Something is wrong with AudioSink, e.g., the device is not
   1096             // connected to audio out.
   1097             // Switch to system clock. This essentially creates a virtual AudioSink with
   1098             // initial latenty of getDurationUsIfPlayedAtSampleRate(mNumFramesWritten).
   1099             // This virtual AudioSink renders audio data starting from the very first sample
   1100             // and it's paced by system clock.
   1101             ALOGW("AudioSink stuck. ARE YOU CONNECTED TO AUDIO OUT? Switching to system clock.");
   1102             mMediaClock->updateAnchor(mAudioFirstAnchorTimeMediaUs, nowUs, mediaTimeUs);
   1103             mUseVirtualAudioSink = true;
   1104         }
   1105     }
   1106     mAnchorNumFramesWritten = mNumFramesWritten;
   1107     mAnchorTimeMediaUs = mediaTimeUs;
   1108 }
   1109 
   1110 // Called without mLock acquired.
   1111 void NuPlayer::Renderer::postDrainVideoQueue() {
   1112     if (mDrainVideoQueuePending
   1113             || getSyncQueues()
   1114             || (mPaused && mVideoSampleReceived)) {
   1115         return;
   1116     }
   1117 
   1118     if (mVideoQueue.empty()) {
   1119         return;
   1120     }
   1121 
   1122     QueueEntry &entry = *mVideoQueue.begin();
   1123 
   1124     sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this);
   1125     msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */));
   1126 
   1127     if (entry.mBuffer == NULL) {
   1128         // EOS doesn't carry a timestamp.
   1129         msg->post();
   1130         mDrainVideoQueuePending = true;
   1131         return;
   1132     }
   1133 
   1134     bool needRepostDrainVideoQueue = false;
   1135     int64_t delayUs;
   1136     int64_t nowUs = ALooper::GetNowUs();
   1137     int64_t realTimeUs;
   1138     if (mFlags & FLAG_REAL_TIME) {
   1139         int64_t mediaTimeUs;
   1140         CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
   1141         realTimeUs = mediaTimeUs;
   1142     } else {
   1143         int64_t mediaTimeUs;
   1144         CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
   1145 
   1146         {
   1147             Mutex::Autolock autoLock(mLock);
   1148             if (mAnchorTimeMediaUs < 0) {
   1149                 mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs);
   1150                 mAnchorTimeMediaUs = mediaTimeUs;
   1151                 realTimeUs = nowUs;
   1152             } else if (!mVideoSampleReceived) {
   1153                 // Always render the first video frame.
   1154                 realTimeUs = nowUs;
   1155             } else if (mAudioFirstAnchorTimeMediaUs < 0
   1156                 || mMediaClock->getRealTimeFor(mediaTimeUs, &realTimeUs) == OK) {
   1157                 realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
   1158             } else if (mediaTimeUs - mAudioFirstAnchorTimeMediaUs >= 0) {
   1159                 needRepostDrainVideoQueue = true;
   1160                 realTimeUs = nowUs;
   1161             } else {
   1162                 realTimeUs = nowUs;
   1163             }
   1164         }
   1165         if (!mHasAudio) {
   1166             // smooth out videos >= 10fps
   1167             mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
   1168         }
   1169 
   1170         // Heuristics to handle situation when media time changed without a
   1171         // discontinuity. If we have not drained an audio buffer that was
   1172         // received after this buffer, repost in 10 msec. Otherwise repost
   1173         // in 500 msec.
   1174         delayUs = realTimeUs - nowUs;
   1175         int64_t postDelayUs = -1;
   1176         if (delayUs > 500000) {
   1177             postDelayUs = 500000;
   1178             if (mHasAudio && (mLastAudioBufferDrained - entry.mBufferOrdinal) <= 0) {
   1179                 postDelayUs = 10000;
   1180             }
   1181         } else if (needRepostDrainVideoQueue) {
   1182             // CHECK(mPlaybackRate > 0);
   1183             // CHECK(mAudioFirstAnchorTimeMediaUs >= 0);
   1184             // CHECK(mediaTimeUs - mAudioFirstAnchorTimeMediaUs >= 0);
   1185             postDelayUs = mediaTimeUs - mAudioFirstAnchorTimeMediaUs;
   1186             postDelayUs /= mPlaybackRate;
   1187         }
   1188 
   1189         if (postDelayUs >= 0) {
   1190             msg->setWhat(kWhatPostDrainVideoQueue);
   1191             msg->post(postDelayUs);
   1192             mVideoScheduler->restart();
   1193             ALOGI("possible video time jump of %dms or uninitialized media clock, retrying in %dms",
   1194                     (int)(delayUs / 1000), (int)(postDelayUs / 1000));
   1195             mDrainVideoQueuePending = true;
   1196             return;
   1197         }
   1198     }
   1199 
   1200     realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
   1201     int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
   1202 
   1203     delayUs = realTimeUs - nowUs;
   1204 
   1205     ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
   1206     // post 2 display refreshes before rendering is due
   1207     msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
   1208 
   1209     mDrainVideoQueuePending = true;
   1210 }
   1211 
   1212 void NuPlayer::Renderer::onDrainVideoQueue() {
   1213     if (mVideoQueue.empty()) {
   1214         return;
   1215     }
   1216 
   1217     QueueEntry *entry = &*mVideoQueue.begin();
   1218 
   1219     if (entry->mBuffer == NULL) {
   1220         // EOS
   1221 
   1222         notifyEOS(false /* audio */, entry->mFinalResult);
   1223 
   1224         mVideoQueue.erase(mVideoQueue.begin());
   1225         entry = NULL;
   1226 
   1227         setVideoLateByUs(0);
   1228         return;
   1229     }
   1230 
   1231     int64_t nowUs = ALooper::GetNowUs();
   1232     int64_t realTimeUs;
   1233     int64_t mediaTimeUs = -1;
   1234     if (mFlags & FLAG_REAL_TIME) {
   1235         CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
   1236     } else {
   1237         CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
   1238 
   1239         realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
   1240     }
   1241 
   1242     bool tooLate = false;
   1243 
   1244     if (!mPaused) {
   1245         setVideoLateByUs(nowUs - realTimeUs);
   1246         tooLate = (mVideoLateByUs > 40000);
   1247 
   1248         if (tooLate) {
   1249             ALOGV("video late by %lld us (%.2f secs)",
   1250                  (long long)mVideoLateByUs, mVideoLateByUs / 1E6);
   1251         } else {
   1252             int64_t mediaUs = 0;
   1253             mMediaClock->getMediaTime(realTimeUs, &mediaUs);
   1254             ALOGV("rendering video at media time %.2f secs",
   1255                     (mFlags & FLAG_REAL_TIME ? realTimeUs :
   1256                     mediaUs) / 1E6);
   1257 
   1258             if (!(mFlags & FLAG_REAL_TIME)
   1259                     && mLastAudioMediaTimeUs != -1
   1260                     && mediaTimeUs > mLastAudioMediaTimeUs) {
   1261                 // If audio ends before video, video continues to drive media clock.
   1262                 // Also smooth out videos >= 10fps.
   1263                 mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
   1264             }
   1265         }
   1266     } else {
   1267         setVideoLateByUs(0);
   1268         if (!mVideoSampleReceived && !mHasAudio) {
   1269             // This will ensure that the first frame after a flush won't be used as anchor
   1270             // when renderer is in paused state, because resume can happen any time after seek.
   1271             Mutex::Autolock autoLock(mLock);
   1272             clearAnchorTime_l();
   1273         }
   1274     }
   1275 
   1276     // Always render the first video frame while keeping stats on A/V sync.
   1277     if (!mVideoSampleReceived) {
   1278         realTimeUs = nowUs;
   1279         tooLate = false;
   1280     }
   1281 
   1282     entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll);
   1283     entry->mNotifyConsumed->setInt32("render", !tooLate);
   1284     entry->mNotifyConsumed->post();
   1285     mVideoQueue.erase(mVideoQueue.begin());
   1286     entry = NULL;
   1287 
   1288     mVideoSampleReceived = true;
   1289 
   1290     if (!mPaused) {
   1291         if (!mVideoRenderingStarted) {
   1292             mVideoRenderingStarted = true;
   1293             notifyVideoRenderingStart();
   1294         }
   1295         Mutex::Autolock autoLock(mLock);
   1296         notifyIfMediaRenderingStarted_l();
   1297     }
   1298 }
   1299 
   1300 void NuPlayer::Renderer::notifyVideoRenderingStart() {
   1301     sp<AMessage> notify = mNotify->dup();
   1302     notify->setInt32("what", kWhatVideoRenderingStart);
   1303     notify->post();
   1304 }
   1305 
   1306 void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) {
   1307     if (audio && delayUs > 0) {
   1308         sp<AMessage> msg = new AMessage(kWhatEOS, this);
   1309         msg->setInt32("audioEOSGeneration", mAudioEOSGeneration);
   1310         msg->setInt32("finalResult", finalResult);
   1311         msg->post(delayUs);
   1312         return;
   1313     }
   1314     sp<AMessage> notify = mNotify->dup();
   1315     notify->setInt32("what", kWhatEOS);
   1316     notify->setInt32("audio", static_cast<int32_t>(audio));
   1317     notify->setInt32("finalResult", finalResult);
   1318     notify->post(delayUs);
   1319 }
   1320 
   1321 void NuPlayer::Renderer::notifyAudioTearDown(AudioTearDownReason reason) {
   1322     sp<AMessage> msg = new AMessage(kWhatAudioTearDown, this);
   1323     msg->setInt32("reason", reason);
   1324     msg->post();
   1325 }
   1326 
   1327 void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
   1328     int32_t audio;
   1329     CHECK(msg->findInt32("audio", &audio));
   1330 
   1331     if (dropBufferIfStale(audio, msg)) {
   1332         return;
   1333     }
   1334 
   1335     if (audio) {
   1336         mHasAudio = true;
   1337     } else {
   1338         mHasVideo = true;
   1339     }
   1340 
   1341     if (mHasVideo) {
   1342         if (mVideoScheduler == NULL) {
   1343             mVideoScheduler = new VideoFrameScheduler();
   1344             mVideoScheduler->init();
   1345         }
   1346     }
   1347 
   1348     sp<ABuffer> buffer;
   1349     CHECK(msg->findBuffer("buffer", &buffer));
   1350 
   1351     sp<AMessage> notifyConsumed;
   1352     CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
   1353 
   1354     QueueEntry entry;
   1355     entry.mBuffer = buffer;
   1356     entry.mNotifyConsumed = notifyConsumed;
   1357     entry.mOffset = 0;
   1358     entry.mFinalResult = OK;
   1359     entry.mBufferOrdinal = ++mTotalBuffersQueued;
   1360 
   1361     if (audio) {
   1362         Mutex::Autolock autoLock(mLock);
   1363         mAudioQueue.push_back(entry);
   1364         postDrainAudioQueue_l();
   1365     } else {
   1366         mVideoQueue.push_back(entry);
   1367         postDrainVideoQueue();
   1368     }
   1369 
   1370     Mutex::Autolock autoLock(mLock);
   1371     if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
   1372         return;
   1373     }
   1374 
   1375     sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
   1376     sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
   1377 
   1378     if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
   1379         // EOS signalled on either queue.
   1380         syncQueuesDone_l();
   1381         return;
   1382     }
   1383 
   1384     int64_t firstAudioTimeUs;
   1385     int64_t firstVideoTimeUs;
   1386     CHECK(firstAudioBuffer->meta()
   1387             ->findInt64("timeUs", &firstAudioTimeUs));
   1388     CHECK(firstVideoBuffer->meta()
   1389             ->findInt64("timeUs", &firstVideoTimeUs));
   1390 
   1391     int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
   1392 
   1393     ALOGV("queueDiff = %.2f secs", diff / 1E6);
   1394 
   1395     if (diff > 100000ll) {
   1396         // Audio data starts More than 0.1 secs before video.
   1397         // Drop some audio.
   1398 
   1399         (*mAudioQueue.begin()).mNotifyConsumed->post();
   1400         mAudioQueue.erase(mAudioQueue.begin());
   1401         return;
   1402     }
   1403 
   1404     syncQueuesDone_l();
   1405 }
   1406 
   1407 void NuPlayer::Renderer::syncQueuesDone_l() {
   1408     if (!mSyncQueues) {
   1409         return;
   1410     }
   1411 
   1412     mSyncQueues = false;
   1413 
   1414     if (!mAudioQueue.empty()) {
   1415         postDrainAudioQueue_l();
   1416     }
   1417 
   1418     if (!mVideoQueue.empty()) {
   1419         mLock.unlock();
   1420         postDrainVideoQueue();
   1421         mLock.lock();
   1422     }
   1423 }
   1424 
   1425 void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
   1426     int32_t audio;
   1427     CHECK(msg->findInt32("audio", &audio));
   1428 
   1429     if (dropBufferIfStale(audio, msg)) {
   1430         return;
   1431     }
   1432 
   1433     int32_t finalResult;
   1434     CHECK(msg->findInt32("finalResult", &finalResult));
   1435 
   1436     QueueEntry entry;
   1437     entry.mOffset = 0;
   1438     entry.mFinalResult = finalResult;
   1439 
   1440     if (audio) {
   1441         Mutex::Autolock autoLock(mLock);
   1442         if (mAudioQueue.empty() && mSyncQueues) {
   1443             syncQueuesDone_l();
   1444         }
   1445         mAudioQueue.push_back(entry);
   1446         postDrainAudioQueue_l();
   1447     } else {
   1448         if (mVideoQueue.empty() && getSyncQueues()) {
   1449             Mutex::Autolock autoLock(mLock);
   1450             syncQueuesDone_l();
   1451         }
   1452         mVideoQueue.push_back(entry);
   1453         postDrainVideoQueue();
   1454     }
   1455 }
   1456 
   1457 void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
   1458     int32_t audio, notifyComplete;
   1459     CHECK(msg->findInt32("audio", &audio));
   1460 
   1461     {
   1462         Mutex::Autolock autoLock(mLock);
   1463         if (audio) {
   1464             notifyComplete = mNotifyCompleteAudio;
   1465             mNotifyCompleteAudio = false;
   1466             mLastAudioMediaTimeUs = -1;
   1467         } else {
   1468             notifyComplete = mNotifyCompleteVideo;
   1469             mNotifyCompleteVideo = false;
   1470         }
   1471 
   1472         // If we're currently syncing the queues, i.e. dropping audio while
   1473         // aligning the first audio/video buffer times and only one of the
   1474         // two queues has data, we may starve that queue by not requesting
   1475         // more buffers from the decoder. If the other source then encounters
   1476         // a discontinuity that leads to flushing, we'll never find the
   1477         // corresponding discontinuity on the other queue.
   1478         // Therefore we'll stop syncing the queues if at least one of them
   1479         // is flushed.
   1480         syncQueuesDone_l();
   1481         clearAnchorTime_l();
   1482     }
   1483 
   1484     ALOGV("flushing %s", audio ? "audio" : "video");
   1485     if (audio) {
   1486         {
   1487             Mutex::Autolock autoLock(mLock);
   1488             flushQueue(&mAudioQueue);
   1489 
   1490             ++mAudioDrainGeneration;
   1491             ++mAudioEOSGeneration;
   1492             prepareForMediaRenderingStart_l();
   1493 
   1494             // the frame count will be reset after flush.
   1495             clearAudioFirstAnchorTime_l();
   1496         }
   1497 
   1498         mDrainAudioQueuePending = false;
   1499 
   1500         if (offloadingAudio()) {
   1501             mAudioSink->pause();
   1502             mAudioSink->flush();
   1503             if (!mPaused) {
   1504                 mAudioSink->start();
   1505             }
   1506         } else {
   1507             mAudioSink->pause();
   1508             mAudioSink->flush();
   1509             // Call stop() to signal to the AudioSink to completely fill the
   1510             // internal buffer before resuming playback.
   1511             // FIXME: this is ignored after flush().
   1512             mAudioSink->stop();
   1513             if (mPaused) {
   1514                 // Race condition: if renderer is paused and audio sink is stopped,
   1515                 // we need to make sure that the audio track buffer fully drains
   1516                 // before delivering data.
   1517                 // FIXME: remove this if we can detect if stop() is complete.
   1518                 const int delayUs = 2 * 50 * 1000; // (2 full mixer thread cycles at 50ms)
   1519                 mPauseDrainAudioAllowedUs = ALooper::GetNowUs() + delayUs;
   1520             } else {
   1521                 mAudioSink->start();
   1522             }
   1523             mNumFramesWritten = 0;
   1524         }
   1525         mNextAudioClockUpdateTimeUs = -1;
   1526     } else {
   1527         flushQueue(&mVideoQueue);
   1528 
   1529         mDrainVideoQueuePending = false;
   1530 
   1531         if (mVideoScheduler != NULL) {
   1532             mVideoScheduler->restart();
   1533         }
   1534 
   1535         Mutex::Autolock autoLock(mLock);
   1536         ++mVideoDrainGeneration;
   1537         prepareForMediaRenderingStart_l();
   1538     }
   1539 
   1540     mVideoSampleReceived = false;
   1541 
   1542     if (notifyComplete) {
   1543         notifyFlushComplete(audio);
   1544     }
   1545 }
   1546 
   1547 void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
   1548     while (!queue->empty()) {
   1549         QueueEntry *entry = &*queue->begin();
   1550 
   1551         if (entry->mBuffer != NULL) {
   1552             entry->mNotifyConsumed->post();
   1553         }
   1554 
   1555         queue->erase(queue->begin());
   1556         entry = NULL;
   1557     }
   1558 }
   1559 
   1560 void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
   1561     sp<AMessage> notify = mNotify->dup();
   1562     notify->setInt32("what", kWhatFlushComplete);
   1563     notify->setInt32("audio", static_cast<int32_t>(audio));
   1564     notify->post();
   1565 }
   1566 
   1567 bool NuPlayer::Renderer::dropBufferIfStale(
   1568         bool audio, const sp<AMessage> &msg) {
   1569     int32_t queueGeneration;
   1570     CHECK(msg->findInt32("queueGeneration", &queueGeneration));
   1571 
   1572     if (queueGeneration == getQueueGeneration(audio)) {
   1573         return false;
   1574     }
   1575 
   1576     sp<AMessage> notifyConsumed;
   1577     if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
   1578         notifyConsumed->post();
   1579     }
   1580 
   1581     return true;
   1582 }
   1583 
   1584 void NuPlayer::Renderer::onAudioSinkChanged() {
   1585     if (offloadingAudio()) {
   1586         return;
   1587     }
   1588     CHECK(!mDrainAudioQueuePending);
   1589     mNumFramesWritten = 0;
   1590     {
   1591         Mutex::Autolock autoLock(mLock);
   1592         mAnchorNumFramesWritten = -1;
   1593     }
   1594     uint32_t written;
   1595     if (mAudioSink->getFramesWritten(&written) == OK) {
   1596         mNumFramesWritten = written;
   1597     }
   1598 }
   1599 
   1600 void NuPlayer::Renderer::onDisableOffloadAudio() {
   1601     Mutex::Autolock autoLock(mLock);
   1602     mFlags &= ~FLAG_OFFLOAD_AUDIO;
   1603     ++mAudioDrainGeneration;
   1604     if (mAudioRenderingStartGeneration != -1) {
   1605         prepareForMediaRenderingStart_l();
   1606     }
   1607 }
   1608 
   1609 void NuPlayer::Renderer::onEnableOffloadAudio() {
   1610     Mutex::Autolock autoLock(mLock);
   1611     mFlags |= FLAG_OFFLOAD_AUDIO;
   1612     ++mAudioDrainGeneration;
   1613     if (mAudioRenderingStartGeneration != -1) {
   1614         prepareForMediaRenderingStart_l();
   1615     }
   1616 }
   1617 
   1618 void NuPlayer::Renderer::onPause() {
   1619     if (mPaused) {
   1620         return;
   1621     }
   1622 
   1623     {
   1624         Mutex::Autolock autoLock(mLock);
   1625         // we do not increment audio drain generation so that we fill audio buffer during pause.
   1626         ++mVideoDrainGeneration;
   1627         prepareForMediaRenderingStart_l();
   1628         mPaused = true;
   1629         mMediaClock->setPlaybackRate(0.0);
   1630     }
   1631 
   1632     mDrainAudioQueuePending = false;
   1633     mDrainVideoQueuePending = false;
   1634 
   1635     // Note: audio data may not have been decoded, and the AudioSink may not be opened.
   1636     mAudioSink->pause();
   1637     startAudioOffloadPauseTimeout();
   1638 
   1639     ALOGV("now paused audio queue has %zu entries, video has %zu entries",
   1640           mAudioQueue.size(), mVideoQueue.size());
   1641 }
   1642 
   1643 void NuPlayer::Renderer::onResume() {
   1644     if (!mPaused) {
   1645         return;
   1646     }
   1647 
   1648     // Note: audio data may not have been decoded, and the AudioSink may not be opened.
   1649     cancelAudioOffloadPauseTimeout();
   1650     if (mAudioSink->ready()) {
   1651         status_t err = mAudioSink->start();
   1652         if (err != OK) {
   1653             ALOGE("cannot start AudioSink err %d", err);
   1654             notifyAudioTearDown(kDueToError);
   1655         }
   1656     }
   1657 
   1658     {
   1659         Mutex::Autolock autoLock(mLock);
   1660         mPaused = false;
   1661         // rendering started message may have been delayed if we were paused.
   1662         if (mRenderingDataDelivered) {
   1663             notifyIfMediaRenderingStarted_l();
   1664         }
   1665         // configure audiosink as we did not do it when pausing
   1666         if (mAudioSink != NULL && mAudioSink->ready()) {
   1667             mAudioSink->setPlaybackRate(mPlaybackSettings);
   1668         }
   1669 
   1670         mMediaClock->setPlaybackRate(mPlaybackRate);
   1671 
   1672         if (!mAudioQueue.empty()) {
   1673             postDrainAudioQueue_l();
   1674         }
   1675     }
   1676 
   1677     if (!mVideoQueue.empty()) {
   1678         postDrainVideoQueue();
   1679     }
   1680 }
   1681 
   1682 void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
   1683     if (mVideoScheduler == NULL) {
   1684         mVideoScheduler = new VideoFrameScheduler();
   1685     }
   1686     mVideoScheduler->init(fps);
   1687 }
   1688 
   1689 int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) {
   1690     Mutex::Autolock autoLock(mLock);
   1691     return (audio ? mAudioQueueGeneration : mVideoQueueGeneration);
   1692 }
   1693 
   1694 int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) {
   1695     Mutex::Autolock autoLock(mLock);
   1696     return (audio ? mAudioDrainGeneration : mVideoDrainGeneration);
   1697 }
   1698 
   1699 bool NuPlayer::Renderer::getSyncQueues() {
   1700     Mutex::Autolock autoLock(mLock);
   1701     return mSyncQueues;
   1702 }
   1703 
   1704 void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) {
   1705     if (mAudioTornDown) {
   1706         return;
   1707     }
   1708     mAudioTornDown = true;
   1709 
   1710     int64_t currentPositionUs;
   1711     sp<AMessage> notify = mNotify->dup();
   1712     if (getCurrentPosition(&currentPositionUs) == OK) {
   1713         notify->setInt64("positionUs", currentPositionUs);
   1714     }
   1715 
   1716     mAudioSink->stop();
   1717     mAudioSink->flush();
   1718 
   1719     notify->setInt32("what", kWhatAudioTearDown);
   1720     notify->setInt32("reason", reason);
   1721     notify->post();
   1722 }
   1723 
   1724 void NuPlayer::Renderer::startAudioOffloadPauseTimeout() {
   1725     if (offloadingAudio()) {
   1726         mWakeLock->acquire();
   1727         sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this);
   1728         msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration);
   1729         msg->post(kOffloadPauseMaxUs);
   1730     }
   1731 }
   1732 
   1733 void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() {
   1734     // We may have called startAudioOffloadPauseTimeout() without
   1735     // the AudioSink open and with offloadingAudio enabled.
   1736     //
   1737     // When we cancel, it may be that offloadingAudio is subsequently disabled, so regardless
   1738     // we always release the wakelock and increment the pause timeout generation.
   1739     //
   1740     // Note: The acquired wakelock prevents the device from suspending
   1741     // immediately after offload pause (in case a resume happens shortly thereafter).
   1742     mWakeLock->release(true);
   1743     ++mAudioOffloadPauseTimeoutGeneration;
   1744 }
   1745 
   1746 status_t NuPlayer::Renderer::onOpenAudioSink(
   1747         const sp<AMessage> &format,
   1748         bool offloadOnly,
   1749         bool hasVideo,
   1750         uint32_t flags) {
   1751     ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)",
   1752             offloadOnly, offloadingAudio());
   1753     bool audioSinkChanged = false;
   1754 
   1755     int32_t numChannels;
   1756     CHECK(format->findInt32("channel-count", &numChannels));
   1757 
   1758     int32_t channelMask;
   1759     if (!format->findInt32("channel-mask", &channelMask)) {
   1760         // signal to the AudioSink to derive the mask from count.
   1761         channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
   1762     }
   1763 
   1764     int32_t sampleRate;
   1765     CHECK(format->findInt32("sample-rate", &sampleRate));
   1766 
   1767     if (offloadingAudio()) {
   1768         audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
   1769         AString mime;
   1770         CHECK(format->findString("mime", &mime));
   1771         status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
   1772 
   1773         if (err != OK) {
   1774             ALOGE("Couldn't map mime \"%s\" to a valid "
   1775                     "audio_format", mime.c_str());
   1776             onDisableOffloadAudio();
   1777         } else {
   1778             ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
   1779                     mime.c_str(), audioFormat);
   1780 
   1781             int avgBitRate = -1;
   1782             format->findInt32("bitrate", &avgBitRate);
   1783 
   1784             int32_t aacProfile = -1;
   1785             if (audioFormat == AUDIO_FORMAT_AAC
   1786                     && format->findInt32("aac-profile", &aacProfile)) {
   1787                 // Redefine AAC format as per aac profile
   1788                 mapAACProfileToAudioFormat(
   1789                         audioFormat,
   1790                         aacProfile);
   1791             }
   1792 
   1793             audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
   1794             offloadInfo.duration_us = -1;
   1795             format->findInt64(
   1796                     "durationUs", &offloadInfo.duration_us);
   1797             offloadInfo.sample_rate = sampleRate;
   1798             offloadInfo.channel_mask = channelMask;
   1799             offloadInfo.format = audioFormat;
   1800             offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
   1801             offloadInfo.bit_rate = avgBitRate;
   1802             offloadInfo.has_video = hasVideo;
   1803             offloadInfo.is_streaming = true;
   1804 
   1805             if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
   1806                 ALOGV("openAudioSink: no change in offload mode");
   1807                 // no change from previous configuration, everything ok.
   1808                 return OK;
   1809             }
   1810             mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
   1811 
   1812             ALOGV("openAudioSink: try to open AudioSink in offload mode");
   1813             uint32_t offloadFlags = flags;
   1814             offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
   1815             offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
   1816             audioSinkChanged = true;
   1817             mAudioSink->close();
   1818 
   1819             err = mAudioSink->open(
   1820                     sampleRate,
   1821                     numChannels,
   1822                     (audio_channel_mask_t)channelMask,
   1823                     audioFormat,
   1824                     0 /* bufferCount - unused */,
   1825                     &NuPlayer::Renderer::AudioSinkCallback,
   1826                     this,
   1827                     (audio_output_flags_t)offloadFlags,
   1828                     &offloadInfo);
   1829 
   1830             if (err == OK) {
   1831                 err = mAudioSink->setPlaybackRate(mPlaybackSettings);
   1832             }
   1833 
   1834             if (err == OK) {
   1835                 // If the playback is offloaded to h/w, we pass
   1836                 // the HAL some metadata information.
   1837                 // We don't want to do this for PCM because it
   1838                 // will be going through the AudioFlinger mixer
   1839                 // before reaching the hardware.
   1840                 // TODO
   1841                 mCurrentOffloadInfo = offloadInfo;
   1842                 if (!mPaused) { // for preview mode, don't start if paused
   1843                     err = mAudioSink->start();
   1844                 }
   1845                 ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
   1846             }
   1847             if (err != OK) {
   1848                 // Clean up, fall back to non offload mode.
   1849                 mAudioSink->close();
   1850                 onDisableOffloadAudio();
   1851                 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
   1852                 ALOGV("openAudioSink: offload failed");
   1853                 if (offloadOnly) {
   1854                     notifyAudioTearDown(kForceNonOffload);
   1855                 }
   1856             } else {
   1857                 mUseAudioCallback = true;  // offload mode transfers data through callback
   1858                 ++mAudioDrainGeneration;  // discard pending kWhatDrainAudioQueue message.
   1859             }
   1860         }
   1861     }
   1862     if (!offloadOnly && !offloadingAudio()) {
   1863         ALOGV("openAudioSink: open AudioSink in NON-offload mode");
   1864         uint32_t pcmFlags = flags;
   1865         pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
   1866 
   1867         const PcmInfo info = {
   1868                 (audio_channel_mask_t)channelMask,
   1869                 (audio_output_flags_t)pcmFlags,
   1870                 AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat
   1871                 numChannels,
   1872                 sampleRate
   1873         };
   1874         if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) {
   1875             ALOGV("openAudioSink: no change in pcm mode");
   1876             // no change from previous configuration, everything ok.
   1877             return OK;
   1878         }
   1879 
   1880         audioSinkChanged = true;
   1881         mAudioSink->close();
   1882         mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
   1883         // Note: It is possible to set up the callback, but not use it to send audio data.
   1884         // This requires a fix in AudioSink to explicitly specify the transfer mode.
   1885         mUseAudioCallback = getUseAudioCallbackSetting();
   1886         if (mUseAudioCallback) {
   1887             ++mAudioDrainGeneration;  // discard pending kWhatDrainAudioQueue message.
   1888         }
   1889 
   1890         // Compute the desired buffer size.
   1891         // For callback mode, the amount of time before wakeup is about half the buffer size.
   1892         const uint32_t frameCount =
   1893                 (unsigned long long)sampleRate * getAudioSinkPcmMsSetting() / 1000;
   1894 
   1895         // The doNotReconnect means AudioSink will signal back and let NuPlayer to re-construct
   1896         // AudioSink. We don't want this when there's video because it will cause a video seek to
   1897         // the previous I frame. But we do want this when there's only audio because it will give
   1898         // NuPlayer a chance to switch from non-offload mode to offload mode.
   1899         // So we only set doNotReconnect when there's no video.
   1900         const bool doNotReconnect = !hasVideo;
   1901 
   1902         // We should always be able to set our playback settings if the sink is closed.
   1903         LOG_ALWAYS_FATAL_IF(mAudioSink->setPlaybackRate(mPlaybackSettings) != OK,
   1904                 "onOpenAudioSink: can't set playback rate on closed sink");
   1905         status_t err = mAudioSink->open(
   1906                     sampleRate,
   1907                     numChannels,
   1908                     (audio_channel_mask_t)channelMask,
   1909                     AUDIO_FORMAT_PCM_16_BIT,
   1910                     0 /* bufferCount - unused */,
   1911                     mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL,
   1912                     mUseAudioCallback ? this : NULL,
   1913                     (audio_output_flags_t)pcmFlags,
   1914                     NULL,
   1915                     doNotReconnect,
   1916                     frameCount);
   1917         if (err != OK) {
   1918             ALOGW("openAudioSink: non offloaded open failed status: %d", err);
   1919             mAudioSink->close();
   1920             mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
   1921             return err;
   1922         }
   1923         mCurrentPcmInfo = info;
   1924         if (!mPaused) { // for preview mode, don't start if paused
   1925             mAudioSink->start();
   1926         }
   1927     }
   1928     if (audioSinkChanged) {
   1929         onAudioSinkChanged();
   1930     }
   1931     mAudioTornDown = false;
   1932     return OK;
   1933 }
   1934 
   1935 void NuPlayer::Renderer::onCloseAudioSink() {
   1936     mAudioSink->close();
   1937     mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
   1938     mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
   1939 }
   1940 
   1941 }  // namespace android
   1942 
   1943