Home | History | Annotate | Download | only in libstagefright 
      1 /*
      2  * Copyright 2015 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 //#define LOG_NDEBUG 0
     18 #define LOG_TAG "MediaSync"
     19 #include <inttypes.h>
     20 
     21 #include <gui/BufferQueue.h>
     22 #include <gui/IGraphicBufferConsumer.h>
     23 #include <gui/IGraphicBufferProducer.h>
     24 
     25 #include <media/AudioTrack.h>
     26 #include <media/stagefright/MediaClock.h>
     27 #include <media/stagefright/MediaSync.h>
     28 #include <media/stagefright/VideoFrameScheduler.h>
     29 #include <media/stagefright/foundation/ADebug.h>
     30 #include <media/stagefright/foundation/ALooper.h>
     31 #include <media/stagefright/foundation/AMessage.h>
     32 
     33 #include <ui/GraphicBuffer.h>
     34 
     35 #include <system/window.h>
     36 
     37 // Maximum late time allowed for a video frame to be rendered. When a video
     38 // frame arrives later than this number, it will be discarded without rendering.
     39 static const int64_t kMaxAllowedVideoLateTimeUs = 40000ll;
     40 
     41 namespace android {
     42 
     43 // static
     44 sp<MediaSync> MediaSync::create() {
     45     sp<MediaSync> sync = new MediaSync();
     46     sync->mLooper->registerHandler(sync);
     47     return sync;
     48 }
     49 
     50 MediaSync::MediaSync()
     51       : mIsAbandoned(false),
     52         mMutex(),
     53         mReleaseCondition(),
     54         mNumOutstandingBuffers(0),
     55         mUsageFlagsFromOutput(0),
     56         mMaxAcquiredBufferCount(1),
     57         mReturnPendingInputFrame(false),
     58         mNativeSampleRateInHz(0),
     59         mNumFramesWritten(0),
     60         mHasAudio(false),
     61         mNextBufferItemMediaUs(-1),
     62         mPlaybackRate(0.0) {
     63     mMediaClock = new MediaClock;
     64     mMediaClock->init();
     65 
     66     // initialize settings
     67     mPlaybackSettings = AUDIO_PLAYBACK_RATE_DEFAULT;
     68     mPlaybackSettings.mSpeed = mPlaybackRate;
     69 
     70     mLooper = new ALooper;
     71     mLooper->setName("MediaSync");
     72     mLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
     73 }
     74 
     75 MediaSync::~MediaSync() {
     76     if (mInput != NULL) {
     77         mInput->consumerDisconnect();
     78     }
     79     if (mOutput != NULL) {
     80         mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
     81     }
     82 
     83     if (mLooper != NULL) {
     84         mLooper->unregisterHandler(id());
     85         mLooper->stop();
     86     }
     87 }
     88 
     89 status_t MediaSync::setSurface(const sp<IGraphicBufferProducer> &output) {
     90     Mutex::Autolock lock(mMutex);
     91 
     92     if (output == mOutput) {
     93         return NO_ERROR;  // same output surface.
     94     }
     95 
     96     if (output == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_VSYNC) {
     97         ALOGE("setSurface: output surface is used as sync source and cannot be removed.");
     98         return INVALID_OPERATION;
     99     }
    100 
    101     if (output != NULL) {
    102         int newUsage = 0;
    103         output->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &newUsage);
    104 
    105         // Check usage flags only when current output surface has been used to create input surface.
    106         if (mOutput != NULL && mInput != NULL) {
    107             int ignoredFlags = (GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_COMPOSER
    108                     | GRALLOC_USAGE_EXTERNAL_DISP);
    109             // New output surface is not allowed to add new usage flag except ignored ones.
    110             if ((newUsage & ~(mUsageFlagsFromOutput | ignoredFlags)) != 0) {
    111                 ALOGE("setSurface: new output surface has new usage flag not used by current one.");
    112                 return BAD_VALUE;
    113             }
    114         }
    115 
    116         // Try to connect to new output surface. If failed, current output surface will not
    117         // be changed.
    118         IGraphicBufferProducer::QueueBufferOutput queueBufferOutput;
    119         sp<OutputListener> listener(new OutputListener(this, output));
    120         IInterface::asBinder(output)->linkToDeath(listener);
    121         status_t status =
    122             output->connect(listener,
    123                             NATIVE_WINDOW_API_MEDIA,
    124                             true /* producerControlledByApp */,
    125                             &queueBufferOutput);
    126         if (status != NO_ERROR) {
    127             ALOGE("setSurface: failed to connect (%d)", status);
    128             return status;
    129         }
    130 
    131         if (mFrameScheduler == NULL) {
    132             mFrameScheduler = new VideoFrameScheduler();
    133             mFrameScheduler->init();
    134         }
    135     }
    136 
    137     if (mOutput != NULL) {
    138         mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
    139         while (!mBuffersSentToOutput.isEmpty()) {
    140             returnBufferToInput_l(mBuffersSentToOutput.valueAt(0), Fence::NO_FENCE);
    141             mBuffersSentToOutput.removeItemsAt(0);
    142         }
    143     }
    144 
    145     mOutput = output;
    146 
    147     return NO_ERROR;
    148 }
    149 
    150 // |audioTrack| is used only for querying information.
    151 status_t MediaSync::setAudioTrack(const sp<AudioTrack> &audioTrack) {
    152     Mutex::Autolock lock(mMutex);
    153 
    154     // TODO: support audio track change.
    155     if (mAudioTrack != NULL) {
    156         ALOGE("setAudioTrack: audioTrack has already been configured.");
    157         return INVALID_OPERATION;
    158     }
    159 
    160     if (audioTrack == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_AUDIO) {
    161         ALOGE("setAudioTrack: audioTrack is used as sync source and cannot be removed.");
    162         return INVALID_OPERATION;
    163     }
    164 
    165     if (audioTrack != NULL) {
    166         // check if audio track supports the playback settings
    167         if (mPlaybackSettings.mSpeed != 0.f
    168                 && audioTrack->setPlaybackRate(mPlaybackSettings) != OK) {
    169             ALOGE("playback settings are not supported by the audio track");
    170             return INVALID_OPERATION;
    171         }
    172         uint32_t nativeSampleRateInHz = audioTrack->getOriginalSampleRate();
    173         if (nativeSampleRateInHz <= 0) {
    174             ALOGE("setAudioTrack: native sample rate should be positive.");
    175             return BAD_VALUE;
    176         }
    177         mAudioTrack = audioTrack;
    178         mNativeSampleRateInHz = nativeSampleRateInHz;
    179         (void)setPlaybackSettings_l(mPlaybackSettings);
    180     }
    181     else {
    182         mAudioTrack = NULL;
    183         mNativeSampleRateInHz = 0;
    184     }
    185 
    186     // potentially resync to new source
    187     resync_l();
    188     return OK;
    189 }
    190 
    191 status_t MediaSync::createInputSurface(
    192         sp<IGraphicBufferProducer> *outBufferProducer) {
    193     if (outBufferProducer == NULL) {
    194         return BAD_VALUE;
    195     }
    196 
    197     Mutex::Autolock lock(mMutex);
    198 
    199     if (mOutput == NULL) {
    200         return NO_INIT;
    201     }
    202 
    203     if (mInput != NULL) {
    204         return INVALID_OPERATION;
    205     }
    206 
    207     sp<IGraphicBufferProducer> bufferProducer;
    208     sp<IGraphicBufferConsumer> bufferConsumer;
    209     BufferQueue::createBufferQueue(&bufferProducer, &bufferConsumer);
    210 
    211     sp<InputListener> listener(new InputListener(this));
    212     IInterface::asBinder(bufferConsumer)->linkToDeath(listener);
    213     status_t status =
    214         bufferConsumer->consumerConnect(listener, false /* controlledByApp */);
    215     if (status == NO_ERROR) {
    216         bufferConsumer->setConsumerName(String8("MediaSync"));
    217         // propagate usage bits from output surface
    218         mUsageFlagsFromOutput = 0;
    219         mOutput->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &mUsageFlagsFromOutput);
    220         bufferConsumer->setConsumerUsageBits(mUsageFlagsFromOutput);
    221         *outBufferProducer = bufferProducer;
    222         mInput = bufferConsumer;
    223 
    224         // set undequeued buffer count
    225         int minUndequeuedBuffers;
    226         mOutput->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBuffers);
    227         mMaxAcquiredBufferCount = minUndequeuedBuffers;
    228         bufferConsumer->setMaxAcquiredBufferCount(mMaxAcquiredBufferCount);
    229     }
    230     return status;
    231 }
    232 
    233 void MediaSync::resync_l() {
    234     AVSyncSource src = mSyncSettings.mSource;
    235     if (src == AVSYNC_SOURCE_DEFAULT) {
    236         if (mAudioTrack != NULL) {
    237             src = AVSYNC_SOURCE_AUDIO;
    238         } else {
    239             src = AVSYNC_SOURCE_SYSTEM_CLOCK;
    240         }
    241     }
    242 
    243     // TODO: resync ourselves to the current clock (e.g. on sync source change)
    244     updatePlaybackRate_l(mPlaybackRate);
    245 }
    246 
    247 void MediaSync::updatePlaybackRate_l(float rate) {
    248     if (rate > mPlaybackRate) {
    249         mNextBufferItemMediaUs = -1;
    250     }
    251     mPlaybackRate = rate;
    252     // TODO: update frame scheduler with this info
    253     mMediaClock->setPlaybackRate(rate);
    254     onDrainVideo_l();
    255 }
    256 
    257 sp<const MediaClock> MediaSync::getMediaClock() {
    258     return mMediaClock;
    259 }
    260 
    261 status_t MediaSync::getPlayTimeForPendingAudioFrames(int64_t *outTimeUs) {
    262     Mutex::Autolock lock(mMutex);
    263     // User should check the playback rate if it doesn't want to receive a
    264     // huge number for play time.
    265     if (mPlaybackRate == 0.0f) {
    266         *outTimeUs = INT64_MAX;
    267         return OK;
    268     }
    269 
    270     uint32_t numFramesPlayed = 0;
    271     if (mAudioTrack != NULL) {
    272         status_t res = mAudioTrack->getPosition(&numFramesPlayed);
    273         if (res != OK) {
    274             return res;
    275         }
    276     }
    277 
    278     int64_t numPendingFrames = mNumFramesWritten - numFramesPlayed;
    279     if (numPendingFrames < 0) {
    280         numPendingFrames = 0;
    281         ALOGW("getPlayTimeForPendingAudioFrames: pending frame count is negative.");
    282     }
    283     double timeUs = numPendingFrames * 1000000.0
    284             / (mNativeSampleRateInHz * (double)mPlaybackRate);
    285     if (timeUs > (double)INT64_MAX) {
    286         // Overflow.
    287         *outTimeUs = INT64_MAX;
    288         ALOGW("getPlayTimeForPendingAudioFrames: play time for pending audio frames "
    289               "is too high, possibly due to super low playback rate(%f)", mPlaybackRate);
    290     } else {
    291         *outTimeUs = (int64_t)timeUs;
    292     }
    293 
    294     return OK;
    295 }
    296 
    297 status_t MediaSync::updateQueuedAudioData(
    298         size_t sizeInBytes, int64_t presentationTimeUs) {
    299     if (sizeInBytes == 0) {
    300         return OK;
    301     }
    302 
    303     Mutex::Autolock lock(mMutex);
    304 
    305     if (mAudioTrack == NULL) {
    306         ALOGW("updateQueuedAudioData: audioTrack has NOT been configured.");
    307         return INVALID_OPERATION;
    308     }
    309 
    310     int64_t numFrames = sizeInBytes / mAudioTrack->frameSize();
    311     int64_t maxMediaTimeUs = presentationTimeUs
    312             + getDurationIfPlayedAtNativeSampleRate_l(numFrames);
    313 
    314     int64_t nowUs = ALooper::GetNowUs();
    315     int64_t nowMediaUs = presentationTimeUs
    316             - getDurationIfPlayedAtNativeSampleRate_l(mNumFramesWritten)
    317             + getPlayedOutAudioDurationMedia_l(nowUs);
    318 
    319     mNumFramesWritten += numFrames;
    320 
    321     int64_t oldRealTime = -1;
    322     if (mNextBufferItemMediaUs != -1) {
    323         oldRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
    324     }
    325 
    326     mMediaClock->updateAnchor(nowMediaUs, nowUs, maxMediaTimeUs);
    327     mHasAudio = true;
    328 
    329     if (oldRealTime != -1) {
    330         int64_t newRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
    331         if (newRealTime >= oldRealTime) {
    332             return OK;
    333         }
    334     }
    335 
    336     mNextBufferItemMediaUs = -1;
    337     onDrainVideo_l();
    338     return OK;
    339 }
    340 
    341 void MediaSync::setName(const AString &name) {
    342     Mutex::Autolock lock(mMutex);
    343     mInput->setConsumerName(String8(name.c_str()));
    344 }
    345 
    346 void MediaSync::flush() {
    347     Mutex::Autolock lock(mMutex);
    348     if (mFrameScheduler != NULL) {
    349         mFrameScheduler->restart();
    350     }
    351     while (!mBufferItems.empty()) {
    352         BufferItem *bufferItem = &*mBufferItems.begin();
    353         returnBufferToInput_l(bufferItem->mGraphicBuffer, bufferItem->mFence);
    354         mBufferItems.erase(mBufferItems.begin());
    355     }
    356     mNextBufferItemMediaUs = -1;
    357     mNumFramesWritten = 0;
    358     mReturnPendingInputFrame = true;
    359     mReleaseCondition.signal();
    360     mMediaClock->clearAnchor();
    361 }
    362 
    363 status_t MediaSync::setVideoFrameRateHint(float rate) {
    364     Mutex::Autolock lock(mMutex);
    365     if (rate < 0.f) {
    366         return BAD_VALUE;
    367     }
    368     if (mFrameScheduler != NULL) {
    369         mFrameScheduler->init(rate);
    370     }
    371     return OK;
    372 }
    373 
    374 float MediaSync::getVideoFrameRate() {
    375     Mutex::Autolock lock(mMutex);
    376     if (mFrameScheduler != NULL) {
    377         float fps = mFrameScheduler->getFrameRate();
    378         if (fps > 0.f) {
    379             return fps;
    380         }
    381     }
    382 
    383     // we don't have or know the frame rate
    384     return -1.f;
    385 }
    386 
    387 status_t MediaSync::setSyncSettings(const AVSyncSettings &syncSettings) {
    388     // validate settings
    389     if (syncSettings.mSource >= AVSYNC_SOURCE_MAX
    390             || syncSettings.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX
    391             || syncSettings.mTolerance < 0.f
    392             || syncSettings.mTolerance >= AVSYNC_TOLERANCE_MAX) {
    393         return BAD_VALUE;
    394     }
    395 
    396     Mutex::Autolock lock(mMutex);
    397 
    398     // verify that we have the sync source
    399     switch (syncSettings.mSource) {
    400         case AVSYNC_SOURCE_AUDIO:
    401             if (mAudioTrack == NULL) {
    402                 ALOGE("setSyncSettings: audio sync source requires an audio track");
    403                 return BAD_VALUE;
    404             }
    405             break;
    406         case AVSYNC_SOURCE_VSYNC:
    407             if (mOutput == NULL) {
    408                 ALOGE("setSyncSettings: vsync sync source requires an output surface");
    409                 return BAD_VALUE;
    410             }
    411             break;
    412         default:
    413             break;
    414     }
    415 
    416     mSyncSettings = syncSettings;
    417     resync_l();
    418     return OK;
    419 }
    420 
    421 void MediaSync::getSyncSettings(AVSyncSettings *syncSettings) {
    422     Mutex::Autolock lock(mMutex);
    423     *syncSettings = mSyncSettings;
    424 }
    425 
    426 status_t MediaSync::setPlaybackSettings(const AudioPlaybackRate &rate) {
    427     Mutex::Autolock lock(mMutex);
    428 
    429     status_t err = setPlaybackSettings_l(rate);
    430     if (err == OK) {
    431         // TODO: adjust rate if using VSYNC as source
    432         updatePlaybackRate_l(rate.mSpeed);
    433     }
    434     return err;
    435 }
    436 
    437 status_t MediaSync::setPlaybackSettings_l(const AudioPlaybackRate &rate) {
    438     if (rate.mSpeed < 0.f || rate.mPitch < 0.f) {
    439         // We don't validate other audio settings.
    440         // They will be validated when/if audiotrack is set.
    441         return BAD_VALUE;
    442     }
    443 
    444     if (mAudioTrack != NULL) {
    445         if (rate.mSpeed == 0.f) {
    446             mAudioTrack->pause();
    447         } else {
    448             status_t err = mAudioTrack->setPlaybackRate(rate);
    449             if (err != OK) {
    450                 return BAD_VALUE;
    451             }
    452 
    453             // ignore errors
    454             (void)mAudioTrack->start();
    455         }
    456     }
    457     mPlaybackSettings = rate;
    458     return OK;
    459 }
    460 
    461 void MediaSync::getPlaybackSettings(AudioPlaybackRate *rate) {
    462     Mutex::Autolock lock(mMutex);
    463     *rate = mPlaybackSettings;
    464 }
    465 
    466 int64_t MediaSync::getRealTime(int64_t mediaTimeUs, int64_t nowUs) {
    467     int64_t realUs;
    468     if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
    469         // If failed to get current position, e.g. due to audio clock is
    470         // not ready, then just play out video immediately without delay.
    471         return nowUs;
    472     }
    473     return realUs;
    474 }
    475 
    476 int64_t MediaSync::getDurationIfPlayedAtNativeSampleRate_l(int64_t numFrames) {
    477     return (numFrames * 1000000LL / mNativeSampleRateInHz);
    478 }
    479 
    480 int64_t MediaSync::getPlayedOutAudioDurationMedia_l(int64_t nowUs) {
    481     CHECK(mAudioTrack != NULL);
    482 
    483     uint32_t numFramesPlayed;
    484     int64_t numFramesPlayedAtUs;
    485     AudioTimestamp ts;
    486 
    487     status_t res = mAudioTrack->getTimestamp(ts);
    488     if (res == OK) {
    489         // case 1: mixing audio tracks.
    490         numFramesPlayed = ts.mPosition;
    491         numFramesPlayedAtUs = ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
    492         //ALOGD("getTimestamp: OK %d %lld",
    493         //      numFramesPlayed, (long long)numFramesPlayedAtUs);
    494     } else if (res == WOULD_BLOCK) {
    495         // case 2: transitory state on start of a new track
    496         numFramesPlayed = 0;
    497         numFramesPlayedAtUs = nowUs;
    498         //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
    499         //      numFramesPlayed, (long long)numFramesPlayedAtUs);
    500     } else {
    501         // case 3: transitory at new track or audio fast tracks.
    502         res = mAudioTrack->getPosition(&numFramesPlayed);
    503         CHECK_EQ(res, (status_t)OK);
    504         numFramesPlayedAtUs = nowUs;
    505         numFramesPlayedAtUs += 1000LL * mAudioTrack->latency() / 2; /* XXX */
    506         //ALOGD("getPosition: %d %lld", numFramesPlayed, (long long)numFramesPlayedAtUs);
    507     }
    508 
    509     //can't be negative until 12.4 hrs, test.
    510     //CHECK_EQ(numFramesPlayed & (1 << 31), 0);
    511     int64_t durationUs =
    512         getDurationIfPlayedAtNativeSampleRate_l(numFramesPlayed)
    513             + nowUs - numFramesPlayedAtUs;
    514     if (durationUs < 0) {
    515         // Occurs when numFramesPlayed position is very small and the following:
    516         // (1) In case 1, the time nowUs is computed before getTimestamp() is
    517         //     called and numFramesPlayedAtUs is greater than nowUs by time more
    518         //     than numFramesPlayed.
    519         // (2) In case 3, using getPosition and adding mAudioTrack->latency()
    520         //     to numFramesPlayedAtUs, by a time amount greater than
    521         //     numFramesPlayed.
    522         //
    523         // Both of these are transitory conditions.
    524         ALOGV("getPlayedOutAudioDurationMedia_l: negative duration %lld "
    525               "set to zero", (long long)durationUs);
    526         durationUs = 0;
    527     }
    528     ALOGV("getPlayedOutAudioDurationMedia_l(%lld) nowUs(%lld) frames(%u) "
    529           "framesAt(%lld)",
    530           (long long)durationUs, (long long)nowUs, numFramesPlayed,
    531           (long long)numFramesPlayedAtUs);
    532     return durationUs;
    533 }
    534 
    535 void MediaSync::onDrainVideo_l() {
    536     if (!isPlaying()) {
    537         return;
    538     }
    539 
    540     while (!mBufferItems.empty()) {
    541         int64_t nowUs = ALooper::GetNowUs();
    542         BufferItem *bufferItem = &*mBufferItems.begin();
    543         int64_t itemMediaUs = bufferItem->mTimestamp / 1000;
    544         int64_t itemRealUs = getRealTime(itemMediaUs, nowUs);
    545 
    546         // adjust video frame PTS based on vsync
    547         itemRealUs = mFrameScheduler->schedule(itemRealUs * 1000) / 1000;
    548         int64_t twoVsyncsUs = 2 * (mFrameScheduler->getVsyncPeriod() / 1000);
    549 
    550         // post 2 display refreshes before rendering is due
    551         if (itemRealUs <= nowUs + twoVsyncsUs) {
    552             ALOGV("adjusting PTS from %lld to %lld",
    553                     (long long)bufferItem->mTimestamp / 1000, (long long)itemRealUs);
    554             bufferItem->mTimestamp = itemRealUs * 1000;
    555             bufferItem->mIsAutoTimestamp = false;
    556 
    557             if (mHasAudio) {
    558                 if (nowUs - itemRealUs <= kMaxAllowedVideoLateTimeUs) {
    559                     renderOneBufferItem_l(*bufferItem);
    560                 } else {
    561                     // too late.
    562                     returnBufferToInput_l(
    563                             bufferItem->mGraphicBuffer, bufferItem->mFence);
    564                     mFrameScheduler->restart();
    565                 }
    566             } else {
    567                 // always render video buffer in video-only mode.
    568                 renderOneBufferItem_l(*bufferItem);
    569 
    570                 // smooth out videos >= 10fps
    571                 mMediaClock->updateAnchor(
    572                         itemMediaUs, nowUs, itemMediaUs + 100000);
    573             }
    574 
    575             mBufferItems.erase(mBufferItems.begin());
    576             mNextBufferItemMediaUs = -1;
    577         } else {
    578             if (mNextBufferItemMediaUs == -1
    579                     || mNextBufferItemMediaUs > itemMediaUs) {
    580                 sp<AMessage> msg = new AMessage(kWhatDrainVideo, this);
    581                 msg->post(itemRealUs - nowUs - twoVsyncsUs);
    582                 mNextBufferItemMediaUs = itemMediaUs;
    583             }
    584             break;
    585         }
    586     }
    587 }
    588 
    589 void MediaSync::onFrameAvailableFromInput() {
    590     Mutex::Autolock lock(mMutex);
    591 
    592     const static nsecs_t kAcquireWaitTimeout = 2000000000; // 2 seconds
    593 
    594     mReturnPendingInputFrame = false;
    595 
    596     // If there are too many outstanding buffers, wait until a buffer is
    597     // released back to the input in onBufferReleased.
    598     // NOTE: BufferQueue allows dequeuing maxAcquiredBufferCount + 1 buffers
    599     while (mNumOutstandingBuffers > mMaxAcquiredBufferCount
    600             && !mIsAbandoned && !mReturnPendingInputFrame) {
    601         if (mReleaseCondition.waitRelative(mMutex, kAcquireWaitTimeout) != OK) {
    602             ALOGI_IF(mPlaybackRate != 0.f, "still waiting to release a buffer before acquire");
    603         }
    604 
    605         // If the sync is abandoned while we are waiting, the release
    606         // condition variable will be broadcast, and we should just return
    607         // without attempting to do anything more (since the input queue will
    608         // also be abandoned).
    609         if (mIsAbandoned) {
    610             return;
    611         }
    612     }
    613 
    614     // Acquire and detach the buffer from the input.
    615     BufferItem bufferItem;
    616     status_t status = mInput->acquireBuffer(&bufferItem, 0 /* presentWhen */);
    617     if (status != NO_ERROR) {
    618         ALOGE("acquiring buffer from input failed (%d)", status);
    619         return;
    620     }
    621     ++mNumOutstandingBuffers;
    622 
    623     ALOGV("acquired buffer %#llx from input", (long long)bufferItem.mGraphicBuffer->getId());
    624 
    625     status = mInput->detachBuffer(bufferItem.mSlot);
    626     if (status != NO_ERROR) {
    627         ALOGE("detaching buffer from input failed (%d)", status);
    628         if (status == NO_INIT) {
    629             // If the input has been abandoned, move on.
    630             onAbandoned_l(true /* isInput */);
    631         }
    632         return;
    633     }
    634 
    635     if (mBuffersFromInput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
    636         // Something is wrong since this buffer should be at our hands, bail.
    637         ALOGE("received buffer multiple times from input");
    638         mInput->consumerDisconnect();
    639         onAbandoned_l(true /* isInput */);
    640         return;
    641     }
    642     mBuffersFromInput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
    643 
    644     // If flush happened while waiting for a buffer to be released, simply return it
    645     // TRICKY: do it here after it is detached so that we don't have to cache mGraphicBuffer.
    646     if (mReturnPendingInputFrame) {
    647         mReturnPendingInputFrame = false;
    648         returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence);
    649         return;
    650     }
    651 
    652     mBufferItems.push_back(bufferItem);
    653 
    654     if (mBufferItems.size() == 1) {
    655         onDrainVideo_l();
    656     }
    657 }
    658 
    659 void MediaSync::renderOneBufferItem_l(const BufferItem &bufferItem) {
    660     IGraphicBufferProducer::QueueBufferInput queueInput(
    661             bufferItem.mTimestamp,
    662             bufferItem.mIsAutoTimestamp,
    663             bufferItem.mDataSpace,
    664             bufferItem.mCrop,
    665             static_cast<int32_t>(bufferItem.mScalingMode),
    666             bufferItem.mTransform,
    667             bufferItem.mFence);
    668 
    669     // Attach and queue the buffer to the output.
    670     int slot;
    671     mOutput->setGenerationNumber(bufferItem.mGraphicBuffer->getGenerationNumber());
    672     status_t status = mOutput->attachBuffer(&slot, bufferItem.mGraphicBuffer);
    673     ALOGE_IF(status != NO_ERROR, "attaching buffer to output failed (%d)", status);
    674     if (status == NO_ERROR) {
    675         IGraphicBufferProducer::QueueBufferOutput queueOutput;
    676         status = mOutput->queueBuffer(slot, queueInput, &queueOutput);
    677         ALOGE_IF(status != NO_ERROR, "queueing buffer to output failed (%d)", status);
    678     }
    679 
    680     if (status != NO_ERROR) {
    681         returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence);
    682         if (status == NO_INIT) {
    683             // If the output has been abandoned, move on.
    684             onAbandoned_l(false /* isInput */);
    685         }
    686         return;
    687     }
    688 
    689     if (mBuffersSentToOutput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
    690         // Something is wrong since this buffer should be held by output now, bail.
    691         mInput->consumerDisconnect();
    692         onAbandoned_l(true /* isInput */);
    693         return;
    694     }
    695     mBuffersSentToOutput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
    696 
    697     ALOGV("queued buffer %#llx to output", (long long)bufferItem.mGraphicBuffer->getId());
    698 }
    699 
    700 void MediaSync::onBufferReleasedByOutput(sp<IGraphicBufferProducer> &output) {
    701     Mutex::Autolock lock(mMutex);
    702 
    703     if (output != mOutput) {
    704         return;  // This is not the current output, ignore.
    705     }
    706 
    707     sp<GraphicBuffer> buffer;
    708     sp<Fence> fence;
    709     status_t status = mOutput->detachNextBuffer(&buffer, &fence);
    710     ALOGE_IF(status != NO_ERROR, "detaching buffer from output failed (%d)", status);
    711 
    712     if (status == NO_INIT) {
    713         // If the output has been abandoned, we can't do anything else,
    714         // since buffer is invalid.
    715         onAbandoned_l(false /* isInput */);
    716         return;
    717     }
    718 
    719     ALOGV("detached buffer %#llx from output", (long long)buffer->getId());
    720 
    721     // If we've been abandoned, we can't return the buffer to the input, so just
    722     // move on.
    723     if (mIsAbandoned) {
    724         return;
    725     }
    726 
    727     ssize_t ix = mBuffersSentToOutput.indexOfKey(buffer->getId());
    728     if (ix < 0) {
    729         // The buffer is unknown, maybe leftover, ignore.
    730         return;
    731     }
    732     mBuffersSentToOutput.removeItemsAt(ix);
    733 
    734     returnBufferToInput_l(buffer, fence);
    735 }
    736 
    737 void MediaSync::returnBufferToInput_l(
    738         const sp<GraphicBuffer> &buffer, const sp<Fence> &fence) {
    739     ssize_t ix = mBuffersFromInput.indexOfKey(buffer->getId());
    740     if (ix < 0) {
    741         // The buffer is unknown, something is wrong, bail.
    742         ALOGE("output returned unknown buffer");
    743         mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
    744         onAbandoned_l(false /* isInput */);
    745         return;
    746     }
    747     sp<GraphicBuffer> oldBuffer = mBuffersFromInput.valueAt(ix);
    748     mBuffersFromInput.removeItemsAt(ix);
    749 
    750     // Attach and release the buffer back to the input.
    751     int consumerSlot;
    752     status_t status = mInput->attachBuffer(&consumerSlot, oldBuffer);
    753     ALOGE_IF(status != NO_ERROR, "attaching buffer to input failed (%d)", status);
    754     if (status == NO_ERROR) {
    755         status = mInput->releaseBuffer(consumerSlot, 0 /* frameNumber */,
    756                 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
    757         ALOGE_IF(status != NO_ERROR, "releasing buffer to input failed (%d)", status);
    758     }
    759 
    760     // Notify any waiting onFrameAvailable calls.
    761     --mNumOutstandingBuffers;
    762     mReleaseCondition.signal();
    763 
    764     if (status == NO_ERROR) {
    765         ALOGV("released buffer %#llx to input", (long long)oldBuffer->getId());
    766     }
    767 }
    768 
    769 void MediaSync::onAbandoned_l(bool isInput) {
    770     ALOGE("the %s has abandoned me", (isInput ? "input" : "output"));
    771     if (!mIsAbandoned) {
    772         if (isInput) {
    773             mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
    774         } else {
    775             mInput->consumerDisconnect();
    776         }
    777         mIsAbandoned = true;
    778     }
    779     mReleaseCondition.broadcast();
    780 }
    781 
    782 void MediaSync::onMessageReceived(const sp<AMessage> &msg) {
    783     switch (msg->what()) {
    784         case kWhatDrainVideo:
    785         {
    786             Mutex::Autolock lock(mMutex);
    787             if (mNextBufferItemMediaUs != -1) {
    788                 int64_t nowUs = ALooper::GetNowUs();
    789                 int64_t itemRealUs = getRealTime(mNextBufferItemMediaUs, nowUs);
    790 
    791                 // The message could arrive earlier than expected due to
    792                 // various reasons, e.g., media clock has been changed because
    793                 // of new anchor time or playback rate. In such cases, the
    794                 // message needs to be re-posted.
    795                 if (itemRealUs > nowUs) {
    796                     msg->post(itemRealUs - nowUs);
    797                     break;
    798                 }
    799             }
    800 
    801             onDrainVideo_l();
    802             break;
    803         }
    804 
    805         default:
    806             TRESPASS();
    807             break;
    808     }
    809 }
    810 
    811 MediaSync::InputListener::InputListener(const sp<MediaSync> &sync)
    812       : mSync(sync) {}
    813 
    814 MediaSync::InputListener::~InputListener() {}
    815 
    816 void MediaSync::InputListener::onFrameAvailable(const BufferItem &/* item */) {
    817     mSync->onFrameAvailableFromInput();
    818 }
    819 
    820 // We don't care about sideband streams, since we won't relay them.
    821 void MediaSync::InputListener::onSidebandStreamChanged() {
    822     ALOGE("onSidebandStreamChanged: got sideband stream unexpectedly.");
    823 }
    824 
    825 
    826 void MediaSync::InputListener::binderDied(const wp<IBinder> &/* who */) {
    827     Mutex::Autolock lock(mSync->mMutex);
    828     mSync->onAbandoned_l(true /* isInput */);
    829 }
    830 
    831 MediaSync::OutputListener::OutputListener(const sp<MediaSync> &sync,
    832         const sp<IGraphicBufferProducer> &output)
    833       : mSync(sync),
    834         mOutput(output) {}
    835 
    836 MediaSync::OutputListener::~OutputListener() {}
    837 
    838 void MediaSync::OutputListener::onBufferReleased() {
    839     mSync->onBufferReleasedByOutput(mOutput);
    840 }
    841 
    842 void MediaSync::OutputListener::binderDied(const wp<IBinder> &/* who */) {
    843     Mutex::Autolock lock(mSync->mMutex);
    844     mSync->onAbandoned_l(false /* isInput */);
    845 }
    846 
    847 } // namespace android
    848