Home | History | Annotate | Download | only in libstagefright 
      1 /*
      2  * Copyright 2015 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 //#define LOG_NDEBUG 0
     18 #define LOG_TAG "MediaSync"
     19 #include <inttypes.h>
     20 
     21 #include <gui/BufferQueue.h>
     22 #include <gui/IGraphicBufferConsumer.h>
     23 #include <gui/IGraphicBufferProducer.h>
     24 
     25 #include <media/AudioTrack.h>
     26 #include <media/stagefright/MediaClock.h>
     27 #include <media/stagefright/MediaSync.h>
     28 #include <media/stagefright/VideoFrameScheduler.h>
     29 #include <media/stagefright/foundation/ADebug.h>
     30 #include <media/stagefright/foundation/ALooper.h>
     31 #include <media/stagefright/foundation/AMessage.h>
     32 
     33 #include <ui/GraphicBuffer.h>
     34 
     35 // Maximum late time allowed for a video frame to be rendered. When a video
     36 // frame arrives later than this number, it will be discarded without rendering.
     37 static const int64_t kMaxAllowedVideoLateTimeUs = 40000ll;
     38 
     39 namespace android {
     40 
     41 // static
     42 sp<MediaSync> MediaSync::create() {
     43     sp<MediaSync> sync = new MediaSync();
     44     sync->mLooper->registerHandler(sync);
     45     return sync;
     46 }
     47 
     48 MediaSync::MediaSync()
     49       : mIsAbandoned(false),
     50         mMutex(),
     51         mReleaseCondition(),
     52         mNumOutstandingBuffers(0),
     53         mUsageFlagsFromOutput(0),
     54         mMaxAcquiredBufferCount(1),
     55         mReturnPendingInputFrame(false),
     56         mNativeSampleRateInHz(0),
     57         mNumFramesWritten(0),
     58         mHasAudio(false),
     59         mNextBufferItemMediaUs(-1),
     60         mPlaybackRate(0.0) {
     61     mMediaClock = new MediaClock;
     62 
     63     // initialize settings
     64     mPlaybackSettings = AUDIO_PLAYBACK_RATE_DEFAULT;
     65     mPlaybackSettings.mSpeed = mPlaybackRate;
     66 
     67     mLooper = new ALooper;
     68     mLooper->setName("MediaSync");
     69     mLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
     70 }
     71 
     72 MediaSync::~MediaSync() {
     73     if (mInput != NULL) {
     74         mInput->consumerDisconnect();
     75     }
     76     if (mOutput != NULL) {
     77         mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
     78     }
     79 
     80     if (mLooper != NULL) {
     81         mLooper->unregisterHandler(id());
     82         mLooper->stop();
     83     }
     84 }
     85 
     86 status_t MediaSync::setSurface(const sp<IGraphicBufferProducer> &output) {
     87     Mutex::Autolock lock(mMutex);
     88 
     89     if (output == mOutput) {
     90         return NO_ERROR;  // same output surface.
     91     }
     92 
     93     if (output == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_VSYNC) {
     94         ALOGE("setSurface: output surface is used as sync source and cannot be removed.");
     95         return INVALID_OPERATION;
     96     }
     97 
     98     if (output != NULL) {
     99         int newUsage = 0;
    100         output->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &newUsage);
    101 
    102         // Check usage flags only when current output surface has been used to create input surface.
    103         if (mOutput != NULL && mInput != NULL) {
    104             int ignoredFlags = (GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_COMPOSER
    105                     | GRALLOC_USAGE_EXTERNAL_DISP);
    106             // New output surface is not allowed to add new usage flag except ignored ones.
    107             if ((newUsage & ~(mUsageFlagsFromOutput | ignoredFlags)) != 0) {
    108                 ALOGE("setSurface: new output surface has new usage flag not used by current one.");
    109                 return BAD_VALUE;
    110             }
    111         }
    112 
    113         // Try to connect to new output surface. If failed, current output surface will not
    114         // be changed.
    115         IGraphicBufferProducer::QueueBufferOutput queueBufferOutput;
    116         sp<OutputListener> listener(new OutputListener(this, output));
    117         IInterface::asBinder(output)->linkToDeath(listener);
    118         status_t status =
    119             output->connect(listener,
    120                             NATIVE_WINDOW_API_MEDIA,
    121                             true /* producerControlledByApp */,
    122                             &queueBufferOutput);
    123         if (status != NO_ERROR) {
    124             ALOGE("setSurface: failed to connect (%d)", status);
    125             return status;
    126         }
    127 
    128         if (mFrameScheduler == NULL) {
    129             mFrameScheduler = new VideoFrameScheduler();
    130             mFrameScheduler->init();
    131         }
    132     }
    133 
    134     if (mOutput != NULL) {
    135         mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
    136         while (!mBuffersSentToOutput.isEmpty()) {
    137             returnBufferToInput_l(mBuffersSentToOutput.valueAt(0), Fence::NO_FENCE);
    138             mBuffersSentToOutput.removeItemsAt(0);
    139         }
    140     }
    141 
    142     mOutput = output;
    143 
    144     return NO_ERROR;
    145 }
    146 
    147 // |audioTrack| is used only for querying information.
    148 status_t MediaSync::setAudioTrack(const sp<AudioTrack> &audioTrack) {
    149     Mutex::Autolock lock(mMutex);
    150 
    151     // TODO: support audio track change.
    152     if (mAudioTrack != NULL) {
    153         ALOGE("setAudioTrack: audioTrack has already been configured.");
    154         return INVALID_OPERATION;
    155     }
    156 
    157     if (audioTrack == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_AUDIO) {
    158         ALOGE("setAudioTrack: audioTrack is used as sync source and cannot be removed.");
    159         return INVALID_OPERATION;
    160     }
    161 
    162     if (audioTrack != NULL) {
    163         // check if audio track supports the playback settings
    164         if (mPlaybackSettings.mSpeed != 0.f
    165                 && audioTrack->setPlaybackRate(mPlaybackSettings) != OK) {
    166             ALOGE("playback settings are not supported by the audio track");
    167             return INVALID_OPERATION;
    168         }
    169         uint32_t nativeSampleRateInHz = audioTrack->getOriginalSampleRate();
    170         if (nativeSampleRateInHz <= 0) {
    171             ALOGE("setAudioTrack: native sample rate should be positive.");
    172             return BAD_VALUE;
    173         }
    174         mAudioTrack = audioTrack;
    175         mNativeSampleRateInHz = nativeSampleRateInHz;
    176         (void)setPlaybackSettings_l(mPlaybackSettings);
    177     }
    178     else {
    179         mAudioTrack = NULL;
    180         mNativeSampleRateInHz = 0;
    181     }
    182 
    183     // potentially resync to new source
    184     resync_l();
    185     return OK;
    186 }
    187 
    188 status_t MediaSync::createInputSurface(
    189         sp<IGraphicBufferProducer> *outBufferProducer) {
    190     if (outBufferProducer == NULL) {
    191         return BAD_VALUE;
    192     }
    193 
    194     Mutex::Autolock lock(mMutex);
    195 
    196     if (mOutput == NULL) {
    197         return NO_INIT;
    198     }
    199 
    200     if (mInput != NULL) {
    201         return INVALID_OPERATION;
    202     }
    203 
    204     sp<IGraphicBufferProducer> bufferProducer;
    205     sp<IGraphicBufferConsumer> bufferConsumer;
    206     BufferQueue::createBufferQueue(&bufferProducer, &bufferConsumer);
    207 
    208     sp<InputListener> listener(new InputListener(this));
    209     IInterface::asBinder(bufferConsumer)->linkToDeath(listener);
    210     status_t status =
    211         bufferConsumer->consumerConnect(listener, false /* controlledByApp */);
    212     if (status == NO_ERROR) {
    213         bufferConsumer->setConsumerName(String8("MediaSync"));
    214         // propagate usage bits from output surface
    215         mUsageFlagsFromOutput = 0;
    216         mOutput->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &mUsageFlagsFromOutput);
    217         bufferConsumer->setConsumerUsageBits(mUsageFlagsFromOutput);
    218         *outBufferProducer = bufferProducer;
    219         mInput = bufferConsumer;
    220 
    221         // set undequeued buffer count
    222         int minUndequeuedBuffers;
    223         mOutput->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBuffers);
    224         mMaxAcquiredBufferCount = minUndequeuedBuffers;
    225         bufferConsumer->setMaxAcquiredBufferCount(mMaxAcquiredBufferCount);
    226     }
    227     return status;
    228 }
    229 
    230 void MediaSync::resync_l() {
    231     AVSyncSource src = mSyncSettings.mSource;
    232     if (src == AVSYNC_SOURCE_DEFAULT) {
    233         if (mAudioTrack != NULL) {
    234             src = AVSYNC_SOURCE_AUDIO;
    235         } else {
    236             src = AVSYNC_SOURCE_SYSTEM_CLOCK;
    237         }
    238     }
    239 
    240     // TODO: resync ourselves to the current clock (e.g. on sync source change)
    241     updatePlaybackRate_l(mPlaybackRate);
    242 }
    243 
    244 void MediaSync::updatePlaybackRate_l(float rate) {
    245     if (rate > mPlaybackRate) {
    246         mNextBufferItemMediaUs = -1;
    247     }
    248     mPlaybackRate = rate;
    249     // TODO: update frame scheduler with this info
    250     mMediaClock->setPlaybackRate(rate);
    251     onDrainVideo_l();
    252 }
    253 
    254 sp<const MediaClock> MediaSync::getMediaClock() {
    255     return mMediaClock;
    256 }
    257 
    258 status_t MediaSync::getPlayTimeForPendingAudioFrames(int64_t *outTimeUs) {
    259     Mutex::Autolock lock(mMutex);
    260     // User should check the playback rate if it doesn't want to receive a
    261     // huge number for play time.
    262     if (mPlaybackRate == 0.0f) {
    263         *outTimeUs = INT64_MAX;
    264         return OK;
    265     }
    266 
    267     uint32_t numFramesPlayed = 0;
    268     if (mAudioTrack != NULL) {
    269         status_t res = mAudioTrack->getPosition(&numFramesPlayed);
    270         if (res != OK) {
    271             return res;
    272         }
    273     }
    274 
    275     int64_t numPendingFrames = mNumFramesWritten - numFramesPlayed;
    276     if (numPendingFrames < 0) {
    277         numPendingFrames = 0;
    278         ALOGW("getPlayTimeForPendingAudioFrames: pending frame count is negative.");
    279     }
    280     double timeUs = numPendingFrames * 1000000.0
    281             / (mNativeSampleRateInHz * (double)mPlaybackRate);
    282     if (timeUs > (double)INT64_MAX) {
    283         // Overflow.
    284         *outTimeUs = INT64_MAX;
    285         ALOGW("getPlayTimeForPendingAudioFrames: play time for pending audio frames "
    286               "is too high, possibly due to super low playback rate(%f)", mPlaybackRate);
    287     } else {
    288         *outTimeUs = (int64_t)timeUs;
    289     }
    290 
    291     return OK;
    292 }
    293 
    294 status_t MediaSync::updateQueuedAudioData(
    295         size_t sizeInBytes, int64_t presentationTimeUs) {
    296     if (sizeInBytes == 0) {
    297         return OK;
    298     }
    299 
    300     Mutex::Autolock lock(mMutex);
    301 
    302     if (mAudioTrack == NULL) {
    303         ALOGW("updateQueuedAudioData: audioTrack has NOT been configured.");
    304         return INVALID_OPERATION;
    305     }
    306 
    307     int64_t numFrames = sizeInBytes / mAudioTrack->frameSize();
    308     int64_t maxMediaTimeUs = presentationTimeUs
    309             + getDurationIfPlayedAtNativeSampleRate_l(numFrames);
    310 
    311     int64_t nowUs = ALooper::GetNowUs();
    312     int64_t nowMediaUs = presentationTimeUs
    313             - getDurationIfPlayedAtNativeSampleRate_l(mNumFramesWritten)
    314             + getPlayedOutAudioDurationMedia_l(nowUs);
    315 
    316     mNumFramesWritten += numFrames;
    317 
    318     int64_t oldRealTime = -1;
    319     if (mNextBufferItemMediaUs != -1) {
    320         oldRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
    321     }
    322 
    323     mMediaClock->updateAnchor(nowMediaUs, nowUs, maxMediaTimeUs);
    324     mHasAudio = true;
    325 
    326     if (oldRealTime != -1) {
    327         int64_t newRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
    328         if (newRealTime >= oldRealTime) {
    329             return OK;
    330         }
    331     }
    332 
    333     mNextBufferItemMediaUs = -1;
    334     onDrainVideo_l();
    335     return OK;
    336 }
    337 
    338 void MediaSync::setName(const AString &name) {
    339     Mutex::Autolock lock(mMutex);
    340     mInput->setConsumerName(String8(name.c_str()));
    341 }
    342 
    343 void MediaSync::flush() {
    344     Mutex::Autolock lock(mMutex);
    345     if (mFrameScheduler != NULL) {
    346         mFrameScheduler->restart();
    347     }
    348     while (!mBufferItems.empty()) {
    349         BufferItem *bufferItem = &*mBufferItems.begin();
    350         returnBufferToInput_l(bufferItem->mGraphicBuffer, bufferItem->mFence);
    351         mBufferItems.erase(mBufferItems.begin());
    352     }
    353     mNextBufferItemMediaUs = -1;
    354     mNumFramesWritten = 0;
    355     mReturnPendingInputFrame = true;
    356     mReleaseCondition.signal();
    357     mMediaClock->clearAnchor();
    358 }
    359 
    360 status_t MediaSync::setVideoFrameRateHint(float rate) {
    361     Mutex::Autolock lock(mMutex);
    362     if (rate < 0.f) {
    363         return BAD_VALUE;
    364     }
    365     if (mFrameScheduler != NULL) {
    366         mFrameScheduler->init(rate);
    367     }
    368     return OK;
    369 }
    370 
    371 float MediaSync::getVideoFrameRate() {
    372     Mutex::Autolock lock(mMutex);
    373     if (mFrameScheduler != NULL) {
    374         float fps = mFrameScheduler->getFrameRate();
    375         if (fps > 0.f) {
    376             return fps;
    377         }
    378     }
    379 
    380     // we don't have or know the frame rate
    381     return -1.f;
    382 }
    383 
    384 status_t MediaSync::setSyncSettings(const AVSyncSettings &syncSettings) {
    385     // validate settings
    386     if (syncSettings.mSource >= AVSYNC_SOURCE_MAX
    387             || syncSettings.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX
    388             || syncSettings.mTolerance < 0.f
    389             || syncSettings.mTolerance >= AVSYNC_TOLERANCE_MAX) {
    390         return BAD_VALUE;
    391     }
    392 
    393     Mutex::Autolock lock(mMutex);
    394 
    395     // verify that we have the sync source
    396     switch (syncSettings.mSource) {
    397         case AVSYNC_SOURCE_AUDIO:
    398             if (mAudioTrack == NULL) {
    399                 ALOGE("setSyncSettings: audio sync source requires an audio track");
    400                 return BAD_VALUE;
    401             }
    402             break;
    403         case AVSYNC_SOURCE_VSYNC:
    404             if (mOutput == NULL) {
    405                 ALOGE("setSyncSettings: vsync sync source requires an output surface");
    406                 return BAD_VALUE;
    407             }
    408             break;
    409         default:
    410             break;
    411     }
    412 
    413     mSyncSettings = syncSettings;
    414     resync_l();
    415     return OK;
    416 }
    417 
    418 void MediaSync::getSyncSettings(AVSyncSettings *syncSettings) {
    419     Mutex::Autolock lock(mMutex);
    420     *syncSettings = mSyncSettings;
    421 }
    422 
    423 status_t MediaSync::setPlaybackSettings(const AudioPlaybackRate &rate) {
    424     Mutex::Autolock lock(mMutex);
    425 
    426     status_t err = setPlaybackSettings_l(rate);
    427     if (err == OK) {
    428         // TODO: adjust rate if using VSYNC as source
    429         updatePlaybackRate_l(rate.mSpeed);
    430     }
    431     return err;
    432 }
    433 
    434 status_t MediaSync::setPlaybackSettings_l(const AudioPlaybackRate &rate) {
    435     if (rate.mSpeed < 0.f || rate.mPitch < 0.f) {
    436         // We don't validate other audio settings.
    437         // They will be validated when/if audiotrack is set.
    438         return BAD_VALUE;
    439     }
    440 
    441     if (mAudioTrack != NULL) {
    442         if (rate.mSpeed == 0.f) {
    443             mAudioTrack->pause();
    444         } else {
    445             status_t err = mAudioTrack->setPlaybackRate(rate);
    446             if (err != OK) {
    447                 return BAD_VALUE;
    448             }
    449 
    450             // ignore errors
    451             (void)mAudioTrack->start();
    452         }
    453     }
    454     mPlaybackSettings = rate;
    455     return OK;
    456 }
    457 
    458 void MediaSync::getPlaybackSettings(AudioPlaybackRate *rate) {
    459     Mutex::Autolock lock(mMutex);
    460     *rate = mPlaybackSettings;
    461 }
    462 
    463 int64_t MediaSync::getRealTime(int64_t mediaTimeUs, int64_t nowUs) {
    464     int64_t realUs;
    465     if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
    466         // If failed to get current position, e.g. due to audio clock is
    467         // not ready, then just play out video immediately without delay.
    468         return nowUs;
    469     }
    470     return realUs;
    471 }
    472 
    473 int64_t MediaSync::getDurationIfPlayedAtNativeSampleRate_l(int64_t numFrames) {
    474     return (numFrames * 1000000LL / mNativeSampleRateInHz);
    475 }
    476 
    477 int64_t MediaSync::getPlayedOutAudioDurationMedia_l(int64_t nowUs) {
    478     CHECK(mAudioTrack != NULL);
    479 
    480     uint32_t numFramesPlayed;
    481     int64_t numFramesPlayedAtUs;
    482     AudioTimestamp ts;
    483 
    484     status_t res = mAudioTrack->getTimestamp(ts);
    485     if (res == OK) {
    486         // case 1: mixing audio tracks.
    487         numFramesPlayed = ts.mPosition;
    488         numFramesPlayedAtUs = ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
    489         //ALOGD("getTimestamp: OK %d %lld",
    490         //      numFramesPlayed, (long long)numFramesPlayedAtUs);
    491     } else if (res == WOULD_BLOCK) {
    492         // case 2: transitory state on start of a new track
    493         numFramesPlayed = 0;
    494         numFramesPlayedAtUs = nowUs;
    495         //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
    496         //      numFramesPlayed, (long long)numFramesPlayedAtUs);
    497     } else {
    498         // case 3: transitory at new track or audio fast tracks.
    499         res = mAudioTrack->getPosition(&numFramesPlayed);
    500         CHECK_EQ(res, (status_t)OK);
    501         numFramesPlayedAtUs = nowUs;
    502         numFramesPlayedAtUs += 1000LL * mAudioTrack->latency() / 2; /* XXX */
    503         //ALOGD("getPosition: %d %lld", numFramesPlayed, (long long)numFramesPlayedAtUs);
    504     }
    505 
    506     //can't be negative until 12.4 hrs, test.
    507     //CHECK_EQ(numFramesPlayed & (1 << 31), 0);
    508     int64_t durationUs =
    509         getDurationIfPlayedAtNativeSampleRate_l(numFramesPlayed)
    510             + nowUs - numFramesPlayedAtUs;
    511     if (durationUs < 0) {
    512         // Occurs when numFramesPlayed position is very small and the following:
    513         // (1) In case 1, the time nowUs is computed before getTimestamp() is
    514         //     called and numFramesPlayedAtUs is greater than nowUs by time more
    515         //     than numFramesPlayed.
    516         // (2) In case 3, using getPosition and adding mAudioTrack->latency()
    517         //     to numFramesPlayedAtUs, by a time amount greater than
    518         //     numFramesPlayed.
    519         //
    520         // Both of these are transitory conditions.
    521         ALOGV("getPlayedOutAudioDurationMedia_l: negative duration %lld "
    522               "set to zero", (long long)durationUs);
    523         durationUs = 0;
    524     }
    525     ALOGV("getPlayedOutAudioDurationMedia_l(%lld) nowUs(%lld) frames(%u) "
    526           "framesAt(%lld)",
    527           (long long)durationUs, (long long)nowUs, numFramesPlayed,
    528           (long long)numFramesPlayedAtUs);
    529     return durationUs;
    530 }
    531 
    532 void MediaSync::onDrainVideo_l() {
    533     if (!isPlaying()) {
    534         return;
    535     }
    536 
    537     while (!mBufferItems.empty()) {
    538         int64_t nowUs = ALooper::GetNowUs();
    539         BufferItem *bufferItem = &*mBufferItems.begin();
    540         int64_t itemMediaUs = bufferItem->mTimestamp / 1000;
    541         int64_t itemRealUs = getRealTime(itemMediaUs, nowUs);
    542 
    543         // adjust video frame PTS based on vsync
    544         itemRealUs = mFrameScheduler->schedule(itemRealUs * 1000) / 1000;
    545         int64_t twoVsyncsUs = 2 * (mFrameScheduler->getVsyncPeriod() / 1000);
    546 
    547         // post 2 display refreshes before rendering is due
    548         if (itemRealUs <= nowUs + twoVsyncsUs) {
    549             ALOGV("adjusting PTS from %lld to %lld",
    550                     (long long)bufferItem->mTimestamp / 1000, (long long)itemRealUs);
    551             bufferItem->mTimestamp = itemRealUs * 1000;
    552             bufferItem->mIsAutoTimestamp = false;
    553 
    554             if (mHasAudio) {
    555                 if (nowUs - itemRealUs <= kMaxAllowedVideoLateTimeUs) {
    556                     renderOneBufferItem_l(*bufferItem);
    557                 } else {
    558                     // too late.
    559                     returnBufferToInput_l(
    560                             bufferItem->mGraphicBuffer, bufferItem->mFence);
    561                     mFrameScheduler->restart();
    562                 }
    563             } else {
    564                 // always render video buffer in video-only mode.
    565                 renderOneBufferItem_l(*bufferItem);
    566 
    567                 // smooth out videos >= 10fps
    568                 mMediaClock->updateAnchor(
    569                         itemMediaUs, nowUs, itemMediaUs + 100000);
    570             }
    571 
    572             mBufferItems.erase(mBufferItems.begin());
    573             mNextBufferItemMediaUs = -1;
    574         } else {
    575             if (mNextBufferItemMediaUs == -1
    576                     || mNextBufferItemMediaUs > itemMediaUs) {
    577                 sp<AMessage> msg = new AMessage(kWhatDrainVideo, this);
    578                 msg->post(itemRealUs - nowUs - twoVsyncsUs);
    579                 mNextBufferItemMediaUs = itemMediaUs;
    580             }
    581             break;
    582         }
    583     }
    584 }
    585 
    586 void MediaSync::onFrameAvailableFromInput() {
    587     Mutex::Autolock lock(mMutex);
    588 
    589     const static nsecs_t kAcquireWaitTimeout = 2000000000; // 2 seconds
    590 
    591     mReturnPendingInputFrame = false;
    592 
    593     // If there are too many outstanding buffers, wait until a buffer is
    594     // released back to the input in onBufferReleased.
    595     // NOTE: BufferQueue allows dequeuing maxAcquiredBufferCount + 1 buffers
    596     while (mNumOutstandingBuffers > mMaxAcquiredBufferCount
    597             && !mIsAbandoned && !mReturnPendingInputFrame) {
    598         if (mReleaseCondition.waitRelative(mMutex, kAcquireWaitTimeout) != OK) {
    599             ALOGI_IF(mPlaybackRate != 0.f, "still waiting to release a buffer before acquire");
    600         }
    601 
    602         // If the sync is abandoned while we are waiting, the release
    603         // condition variable will be broadcast, and we should just return
    604         // without attempting to do anything more (since the input queue will
    605         // also be abandoned).
    606         if (mIsAbandoned) {
    607             return;
    608         }
    609     }
    610 
    611     // Acquire and detach the buffer from the input.
    612     BufferItem bufferItem;
    613     status_t status = mInput->acquireBuffer(&bufferItem, 0 /* presentWhen */);
    614     if (status != NO_ERROR) {
    615         ALOGE("acquiring buffer from input failed (%d)", status);
    616         return;
    617     }
    618     ++mNumOutstandingBuffers;
    619 
    620     ALOGV("acquired buffer %#llx from input", (long long)bufferItem.mGraphicBuffer->getId());
    621 
    622     status = mInput->detachBuffer(bufferItem.mSlot);
    623     if (status != NO_ERROR) {
    624         ALOGE("detaching buffer from input failed (%d)", status);
    625         if (status == NO_INIT) {
    626             // If the input has been abandoned, move on.
    627             onAbandoned_l(true /* isInput */);
    628         }
    629         return;
    630     }
    631 
    632     if (mBuffersFromInput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
    633         // Something is wrong since this buffer should be at our hands, bail.
    634         ALOGE("received buffer multiple times from input");
    635         mInput->consumerDisconnect();
    636         onAbandoned_l(true /* isInput */);
    637         return;
    638     }
    639     mBuffersFromInput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
    640 
    641     // If flush happened while waiting for a buffer to be released, simply return it
    642     // TRICKY: do it here after it is detached so that we don't have to cache mGraphicBuffer.
    643     if (mReturnPendingInputFrame) {
    644         mReturnPendingInputFrame = false;
    645         returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence);
    646         return;
    647     }
    648 
    649     mBufferItems.push_back(bufferItem);
    650 
    651     if (mBufferItems.size() == 1) {
    652         onDrainVideo_l();
    653     }
    654 }
    655 
    656 void MediaSync::renderOneBufferItem_l(const BufferItem &bufferItem) {
    657     IGraphicBufferProducer::QueueBufferInput queueInput(
    658             bufferItem.mTimestamp,
    659             bufferItem.mIsAutoTimestamp,
    660             bufferItem.mDataSpace,
    661             bufferItem.mCrop,
    662             static_cast<int32_t>(bufferItem.mScalingMode),
    663             bufferItem.mTransform,
    664             bufferItem.mFence);
    665 
    666     // Attach and queue the buffer to the output.
    667     int slot;
    668     mOutput->setGenerationNumber(bufferItem.mGraphicBuffer->getGenerationNumber());
    669     status_t status = mOutput->attachBuffer(&slot, bufferItem.mGraphicBuffer);
    670     ALOGE_IF(status != NO_ERROR, "attaching buffer to output failed (%d)", status);
    671     if (status == NO_ERROR) {
    672         IGraphicBufferProducer::QueueBufferOutput queueOutput;
    673         status = mOutput->queueBuffer(slot, queueInput, &queueOutput);
    674         ALOGE_IF(status != NO_ERROR, "queueing buffer to output failed (%d)", status);
    675     }
    676 
    677     if (status != NO_ERROR) {
    678         returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence);
    679         if (status == NO_INIT) {
    680             // If the output has been abandoned, move on.
    681             onAbandoned_l(false /* isInput */);
    682         }
    683         return;
    684     }
    685 
    686     if (mBuffersSentToOutput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
    687         // Something is wrong since this buffer should be held by output now, bail.
    688         mInput->consumerDisconnect();
    689         onAbandoned_l(true /* isInput */);
    690         return;
    691     }
    692     mBuffersSentToOutput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
    693 
    694     ALOGV("queued buffer %#llx to output", (long long)bufferItem.mGraphicBuffer->getId());
    695 }
    696 
    697 void MediaSync::onBufferReleasedByOutput(sp<IGraphicBufferProducer> &output) {
    698     Mutex::Autolock lock(mMutex);
    699 
    700     if (output != mOutput) {
    701         return;  // This is not the current output, ignore.
    702     }
    703 
    704     sp<GraphicBuffer> buffer;
    705     sp<Fence> fence;
    706     status_t status = mOutput->detachNextBuffer(&buffer, &fence);
    707     ALOGE_IF(status != NO_ERROR, "detaching buffer from output failed (%d)", status);
    708 
    709     if (status == NO_INIT) {
    710         // If the output has been abandoned, we can't do anything else,
    711         // since buffer is invalid.
    712         onAbandoned_l(false /* isInput */);
    713         return;
    714     }
    715 
    716     ALOGV("detached buffer %#llx from output", (long long)buffer->getId());
    717 
    718     // If we've been abandoned, we can't return the buffer to the input, so just
    719     // move on.
    720     if (mIsAbandoned) {
    721         return;
    722     }
    723 
    724     ssize_t ix = mBuffersSentToOutput.indexOfKey(buffer->getId());
    725     if (ix < 0) {
    726         // The buffer is unknown, maybe leftover, ignore.
    727         return;
    728     }
    729     mBuffersSentToOutput.removeItemsAt(ix);
    730 
    731     returnBufferToInput_l(buffer, fence);
    732 }
    733 
    734 void MediaSync::returnBufferToInput_l(
    735         const sp<GraphicBuffer> &buffer, const sp<Fence> &fence) {
    736     ssize_t ix = mBuffersFromInput.indexOfKey(buffer->getId());
    737     if (ix < 0) {
    738         // The buffer is unknown, something is wrong, bail.
    739         ALOGE("output returned unknown buffer");
    740         mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
    741         onAbandoned_l(false /* isInput */);
    742         return;
    743     }
    744     sp<GraphicBuffer> oldBuffer = mBuffersFromInput.valueAt(ix);
    745     mBuffersFromInput.removeItemsAt(ix);
    746 
    747     // Attach and release the buffer back to the input.
    748     int consumerSlot;
    749     status_t status = mInput->attachBuffer(&consumerSlot, oldBuffer);
    750     ALOGE_IF(status != NO_ERROR, "attaching buffer to input failed (%d)", status);
    751     if (status == NO_ERROR) {
    752         status = mInput->releaseBuffer(consumerSlot, 0 /* frameNumber */,
    753                 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
    754         ALOGE_IF(status != NO_ERROR, "releasing buffer to input failed (%d)", status);
    755     }
    756 
    757     // Notify any waiting onFrameAvailable calls.
    758     --mNumOutstandingBuffers;
    759     mReleaseCondition.signal();
    760 
    761     if (status == NO_ERROR) {
    762         ALOGV("released buffer %#llx to input", (long long)oldBuffer->getId());
    763     }
    764 }
    765 
    766 void MediaSync::onAbandoned_l(bool isInput) {
    767     ALOGE("the %s has abandoned me", (isInput ? "input" : "output"));
    768     if (!mIsAbandoned) {
    769         if (isInput) {
    770             mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
    771         } else {
    772             mInput->consumerDisconnect();
    773         }
    774         mIsAbandoned = true;
    775     }
    776     mReleaseCondition.broadcast();
    777 }
    778 
    779 void MediaSync::onMessageReceived(const sp<AMessage> &msg) {
    780     switch (msg->what()) {
    781         case kWhatDrainVideo:
    782         {
    783             Mutex::Autolock lock(mMutex);
    784             if (mNextBufferItemMediaUs != -1) {
    785                 int64_t nowUs = ALooper::GetNowUs();
    786                 int64_t itemRealUs = getRealTime(mNextBufferItemMediaUs, nowUs);
    787 
    788                 // The message could arrive earlier than expected due to
    789                 // various reasons, e.g., media clock has been changed because
    790                 // of new anchor time or playback rate. In such cases, the
    791                 // message needs to be re-posted.
    792                 if (itemRealUs > nowUs) {
    793                     msg->post(itemRealUs - nowUs);
    794                     break;
    795                 }
    796             }
    797 
    798             onDrainVideo_l();
    799             break;
    800         }
    801 
    802         default:
    803             TRESPASS();
    804             break;
    805     }
    806 }
    807 
    808 MediaSync::InputListener::InputListener(const sp<MediaSync> &sync)
    809       : mSync(sync) {}
    810 
    811 MediaSync::InputListener::~InputListener() {}
    812 
    813 void MediaSync::InputListener::onFrameAvailable(const BufferItem &/* item */) {
    814     mSync->onFrameAvailableFromInput();
    815 }
    816 
    817 // We don't care about sideband streams, since we won't relay them.
    818 void MediaSync::InputListener::onSidebandStreamChanged() {
    819     ALOGE("onSidebandStreamChanged: got sideband stream unexpectedly.");
    820 }
    821 
    822 
    823 void MediaSync::InputListener::binderDied(const wp<IBinder> &/* who */) {
    824     Mutex::Autolock lock(mSync->mMutex);
    825     mSync->onAbandoned_l(true /* isInput */);
    826 }
    827 
    828 MediaSync::OutputListener::OutputListener(const sp<MediaSync> &sync,
    829         const sp<IGraphicBufferProducer> &output)
    830       : mSync(sync),
    831         mOutput(output) {}
    832 
    833 MediaSync::OutputListener::~OutputListener() {}
    834 
    835 void MediaSync::OutputListener::onBufferReleased() {
    836     mSync->onBufferReleasedByOutput(mOutput);
    837 }
    838 
    839 void MediaSync::OutputListener::binderDied(const wp<IBinder> &/* who */) {
    840     Mutex::Autolock lock(mSync->mMutex);
    841     mSync->onAbandoned_l(false /* isInput */);
    842 }
    843 
    844 } // namespace android
    845