Home | History | Annotate | Download | only in libstagefright 
      1 /*
      2  * Copyright 2015 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 //#define LOG_NDEBUG 0
     18 #define LOG_TAG "MediaSync"
     19 #include <inttypes.h>
     20 
     21 #include <gui/BufferQueue.h>
     22 #include <gui/IGraphicBufferConsumer.h>
     23 #include <gui/IGraphicBufferProducer.h>
     24 
     25 #include <media/AudioTrack.h>
     26 #include <media/stagefright/MediaClock.h>
     27 #include <media/stagefright/MediaSync.h>
     28 #include <media/stagefright/VideoFrameScheduler.h>
     29 #include <media/stagefright/foundation/ADebug.h>
     30 #include <media/stagefright/foundation/ALooper.h>
     31 #include <media/stagefright/foundation/AMessage.h>
     32 
     33 #include <ui/GraphicBuffer.h>
     34 
     35 // Maximum late time allowed for a video frame to be rendered. When a video
     36 // frame arrives later than this number, it will be discarded without rendering.
     37 static const int64_t kMaxAllowedVideoLateTimeUs = 40000ll;
     38 
     39 namespace android {
     40 
     41 // static
     42 sp<MediaSync> MediaSync::create() {
     43     sp<MediaSync> sync = new MediaSync();
     44     sync->mLooper->registerHandler(sync);
     45     return sync;
     46 }
     47 
     48 MediaSync::MediaSync()
     49       : mIsAbandoned(false),
     50         mMutex(),
     51         mReleaseCondition(),
     52         mNumOutstandingBuffers(0),
     53         mUsageFlagsFromOutput(0),
     54         mMaxAcquiredBufferCount(1),
     55         mReturnPendingInputFrame(false),
     56         mNativeSampleRateInHz(0),
     57         mNumFramesWritten(0),
     58         mHasAudio(false),
     59         mNextBufferItemMediaUs(-1),
     60         mPlaybackRate(0.0) {
     61     mMediaClock = new MediaClock;
     62 
     63     // initialize settings
     64     mPlaybackSettings = AUDIO_PLAYBACK_RATE_DEFAULT;
     65     mPlaybackSettings.mSpeed = mPlaybackRate;
     66 
     67     mLooper = new ALooper;
     68     mLooper->setName("MediaSync");
     69     mLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
     70 }
     71 
     72 MediaSync::~MediaSync() {
     73     if (mInput != NULL) {
     74         mInput->consumerDisconnect();
     75     }
     76     if (mOutput != NULL) {
     77         mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
     78     }
     79 
     80     if (mLooper != NULL) {
     81         mLooper->unregisterHandler(id());
     82         mLooper->stop();
     83     }
     84 }
     85 
     86 status_t MediaSync::setSurface(const sp<IGraphicBufferProducer> &output) {
     87     Mutex::Autolock lock(mMutex);
     88 
     89     if (output == mOutput) {
     90         return NO_ERROR;  // same output surface.
     91     }
     92 
     93     if (output == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_VSYNC) {
     94         ALOGE("setSurface: output surface is used as sync source and cannot be removed.");
     95         return INVALID_OPERATION;
     96     }
     97 
     98     if (output != NULL) {
     99         int newUsage = 0;
    100         output->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &newUsage);
    101 
    102         // Check usage flags only when current output surface has been used to create input surface.
    103         if (mOutput != NULL && mInput != NULL) {
    104             int ignoredFlags = (GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_COMPOSER
    105                     | GRALLOC_USAGE_EXTERNAL_DISP);
    106             // New output surface is not allowed to add new usage flag except ignored ones.
    107             if ((newUsage & ~(mUsageFlagsFromOutput | ignoredFlags)) != 0) {
    108                 ALOGE("setSurface: new output surface has new usage flag not used by current one.");
    109                 return BAD_VALUE;
    110             }
    111         }
    112 
    113         // Try to connect to new output surface. If failed, current output surface will not
    114         // be changed.
    115         IGraphicBufferProducer::QueueBufferOutput queueBufferOutput;
    116         sp<OutputListener> listener(new OutputListener(this, output));
    117         IInterface::asBinder(output)->linkToDeath(listener);
    118         status_t status =
    119             output->connect(listener,
    120                             NATIVE_WINDOW_API_MEDIA,
    121                             true /* producerControlledByApp */,
    122                             &queueBufferOutput);
    123         if (status != NO_ERROR) {
    124             ALOGE("setSurface: failed to connect (%d)", status);
    125             return status;
    126         }
    127 
    128         if (mFrameScheduler == NULL) {
    129             mFrameScheduler = new VideoFrameScheduler();
    130             mFrameScheduler->init();
    131         }
    132     }
    133 
    134     if (mOutput != NULL) {
    135         mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
    136         while (!mBuffersSentToOutput.isEmpty()) {
    137             returnBufferToInput_l(mBuffersSentToOutput.valueAt(0), Fence::NO_FENCE);
    138             mBuffersSentToOutput.removeItemsAt(0);
    139         }
    140     }
    141 
    142     mOutput = output;
    143 
    144     return NO_ERROR;
    145 }
    146 
    147 // |audioTrack| is used only for querying information.
    148 status_t MediaSync::setAudioTrack(const sp<AudioTrack> &audioTrack) {
    149     Mutex::Autolock lock(mMutex);
    150 
    151     // TODO: support audio track change.
    152     if (mAudioTrack != NULL) {
    153         ALOGE("setAudioTrack: audioTrack has already been configured.");
    154         return INVALID_OPERATION;
    155     }
    156 
    157     if (audioTrack == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_AUDIO) {
    158         ALOGE("setAudioTrack: audioTrack is used as sync source and cannot be removed.");
    159         return INVALID_OPERATION;
    160     }
    161 
    162     if (audioTrack != NULL) {
    163         // check if audio track supports the playback settings
    164         if (mPlaybackSettings.mSpeed != 0.f
    165                 && audioTrack->setPlaybackRate(mPlaybackSettings) != OK) {
    166             ALOGE("playback settings are not supported by the audio track");
    167             return INVALID_OPERATION;
    168         }
    169         uint32_t nativeSampleRateInHz = audioTrack->getOriginalSampleRate();
    170         if (nativeSampleRateInHz <= 0) {
    171             ALOGE("setAudioTrack: native sample rate should be positive.");
    172             return BAD_VALUE;
    173         }
    174         mAudioTrack = audioTrack;
    175         mNativeSampleRateInHz = nativeSampleRateInHz;
    176         (void)setPlaybackSettings_l(mPlaybackSettings);
    177     }
    178     else {
    179         mAudioTrack = NULL;
    180         mNativeSampleRateInHz = 0;
    181     }
    182 
    183     // potentially resync to new source
    184     resync_l();
    185     return OK;
    186 }
    187 
    188 status_t MediaSync::createInputSurface(
    189         sp<IGraphicBufferProducer> *outBufferProducer) {
    190     if (outBufferProducer == NULL) {
    191         return BAD_VALUE;
    192     }
    193 
    194     Mutex::Autolock lock(mMutex);
    195 
    196     if (mOutput == NULL) {
    197         return NO_INIT;
    198     }
    199 
    200     if (mInput != NULL) {
    201         return INVALID_OPERATION;
    202     }
    203 
    204     sp<IGraphicBufferProducer> bufferProducer;
    205     sp<IGraphicBufferConsumer> bufferConsumer;
    206     BufferQueue::createBufferQueue(&bufferProducer, &bufferConsumer);
    207 
    208     sp<InputListener> listener(new InputListener(this));
    209     IInterface::asBinder(bufferConsumer)->linkToDeath(listener);
    210     status_t status =
    211         bufferConsumer->consumerConnect(listener, false /* controlledByApp */);
    212     if (status == NO_ERROR) {
    213         bufferConsumer->setConsumerName(String8("MediaSync"));
    214         // propagate usage bits from output surface
    215         mUsageFlagsFromOutput = 0;
    216         mOutput->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &mUsageFlagsFromOutput);
    217         bufferConsumer->setConsumerUsageBits(mUsageFlagsFromOutput);
    218         *outBufferProducer = bufferProducer;
    219         mInput = bufferConsumer;
    220 
    221         // set undequeued buffer count
    222         int minUndequeuedBuffers;
    223         mOutput->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBuffers);
    224         mMaxAcquiredBufferCount = minUndequeuedBuffers;
    225         bufferConsumer->setMaxAcquiredBufferCount(mMaxAcquiredBufferCount);
    226     }
    227     return status;
    228 }
    229 
    230 void MediaSync::resync_l() {
    231     AVSyncSource src = mSyncSettings.mSource;
    232     if (src == AVSYNC_SOURCE_DEFAULT) {
    233         if (mAudioTrack != NULL) {
    234             src = AVSYNC_SOURCE_AUDIO;
    235         } else {
    236             src = AVSYNC_SOURCE_SYSTEM_CLOCK;
    237         }
    238     }
    239 
    240     // TODO: resync ourselves to the current clock (e.g. on sync source change)
    241     updatePlaybackRate_l(mPlaybackRate);
    242 }
    243 
    244 void MediaSync::updatePlaybackRate_l(float rate) {
    245     if (rate > mPlaybackRate) {
    246         mNextBufferItemMediaUs = -1;
    247     }
    248     mPlaybackRate = rate;
    249     // TODO: update frame scheduler with this info
    250     mMediaClock->setPlaybackRate(rate);
    251     onDrainVideo_l();
    252 }
    253 
    254 sp<const MediaClock> MediaSync::getMediaClock() {
    255     return mMediaClock;
    256 }
    257 
    258 status_t MediaSync::getPlayTimeForPendingAudioFrames(int64_t *outTimeUs) {
    259     Mutex::Autolock lock(mMutex);
    260     // User should check the playback rate if it doesn't want to receive a
    261     // huge number for play time.
    262     if (mPlaybackRate == 0.0f) {
    263         *outTimeUs = INT64_MAX;
    264         return OK;
    265     }
    266 
    267     uint32_t numFramesPlayed = 0;
    268     if (mAudioTrack != NULL) {
    269         status_t res = mAudioTrack->getPosition(&numFramesPlayed);
    270         if (res != OK) {
    271             return res;
    272         }
    273     }
    274 
    275     int64_t numPendingFrames = mNumFramesWritten - numFramesPlayed;
    276     if (numPendingFrames < 0) {
    277         numPendingFrames = 0;
    278         ALOGW("getPlayTimeForPendingAudioFrames: pending frame count is negative.");
    279     }
    280     double timeUs = numPendingFrames * 1000000.0
    281             / (mNativeSampleRateInHz * (double)mPlaybackRate);
    282     if (timeUs > (double)INT64_MAX) {
    283         // Overflow.
    284         *outTimeUs = INT64_MAX;
    285         ALOGW("getPlayTimeForPendingAudioFrames: play time for pending audio frames "
    286               "is too high, possibly due to super low playback rate(%f)", mPlaybackRate);
    287     } else {
    288         *outTimeUs = (int64_t)timeUs;
    289     }
    290 
    291     return OK;
    292 }
    293 
    294 status_t MediaSync::updateQueuedAudioData(
    295         size_t sizeInBytes, int64_t presentationTimeUs) {
    296     if (sizeInBytes == 0) {
    297         return OK;
    298     }
    299 
    300     Mutex::Autolock lock(mMutex);
    301 
    302     if (mAudioTrack == NULL) {
    303         ALOGW("updateQueuedAudioData: audioTrack has NOT been configured.");
    304         return INVALID_OPERATION;
    305     }
    306 
    307     int64_t numFrames = sizeInBytes / mAudioTrack->frameSize();
    308     int64_t maxMediaTimeUs = presentationTimeUs
    309             + getDurationIfPlayedAtNativeSampleRate_l(numFrames);
    310 
    311     int64_t nowUs = ALooper::GetNowUs();
    312     int64_t nowMediaUs = presentationTimeUs
    313             - getDurationIfPlayedAtNativeSampleRate_l(mNumFramesWritten)
    314             + getPlayedOutAudioDurationMedia_l(nowUs);
    315 
    316     mNumFramesWritten += numFrames;
    317 
    318     int64_t oldRealTime = -1;
    319     if (mNextBufferItemMediaUs != -1) {
    320         oldRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
    321     }
    322 
    323     mMediaClock->updateAnchor(nowMediaUs, nowUs, maxMediaTimeUs);
    324     mHasAudio = true;
    325 
    326     if (oldRealTime != -1) {
    327         int64_t newRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
    328         if (newRealTime >= oldRealTime) {
    329             return OK;
    330         }
    331     }
    332 
    333     mNextBufferItemMediaUs = -1;
    334     onDrainVideo_l();
    335     return OK;
    336 }
    337 
    338 void MediaSync::setName(const AString &name) {
    339     Mutex::Autolock lock(mMutex);
    340     mInput->setConsumerName(String8(name.c_str()));
    341 }
    342 
    343 void MediaSync::flush() {
    344     Mutex::Autolock lock(mMutex);
    345     if (mFrameScheduler != NULL) {
    346         mFrameScheduler->restart();
    347     }
    348     while (!mBufferItems.empty()) {
    349         BufferItem *bufferItem = &*mBufferItems.begin();
    350         returnBufferToInput_l(bufferItem->mGraphicBuffer, bufferItem->mFence);
    351         mBufferItems.erase(mBufferItems.begin());
    352     }
    353     mNextBufferItemMediaUs = -1;
    354     mNumFramesWritten = 0;
    355     mReturnPendingInputFrame = true;
    356     mReleaseCondition.signal();
    357     mMediaClock->clearAnchor();
    358 }
    359 
    360 status_t MediaSync::setVideoFrameRateHint(float rate) {
    361     Mutex::Autolock lock(mMutex);
    362     if (rate < 0.f) {
    363         return BAD_VALUE;
    364     }
    365     if (mFrameScheduler != NULL) {
    366         mFrameScheduler->init(rate);
    367     }
    368     return OK;
    369 }
    370 
    371 float MediaSync::getVideoFrameRate() {
    372     Mutex::Autolock lock(mMutex);
    373     if (mFrameScheduler != NULL) {
    374         float fps = mFrameScheduler->getFrameRate();
    375         if (fps > 0.f) {
    376             return fps;
    377         }
    378     }
    379 
    380     // we don't have or know the frame rate
    381     return -1.f;
    382 }
    383 
    384 status_t MediaSync::setSyncSettings(const AVSyncSettings &syncSettings) {
    385     // validate settings
    386     if (syncSettings.mSource >= AVSYNC_SOURCE_MAX
    387             || syncSettings.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX
    388             || syncSettings.mTolerance < 0.f
    389             || syncSettings.mTolerance >= AVSYNC_TOLERANCE_MAX) {
    390         return BAD_VALUE;
    391     }
    392 
    393     Mutex::Autolock lock(mMutex);
    394 
    395     // verify that we have the sync source
    396     switch (syncSettings.mSource) {
    397         case AVSYNC_SOURCE_AUDIO:
    398             if (mAudioTrack == NULL) {
    399                 ALOGE("setSyncSettings: audio sync source requires an audio track");
    400                 return BAD_VALUE;
    401             }
    402             break;
    403         case AVSYNC_SOURCE_VSYNC:
    404             if (mOutput == NULL) {
    405                 ALOGE("setSyncSettings: vsync sync source requires an output surface");
    406                 return BAD_VALUE;
    407             }
    408             break;
    409         default:
    410             break;
    411     }
    412 
    413     mSyncSettings = syncSettings;
    414     resync_l();
    415     return OK;
    416 }
    417 
    418 void MediaSync::getSyncSettings(AVSyncSettings *syncSettings) {
    419     Mutex::Autolock lock(mMutex);
    420     *syncSettings = mSyncSettings;
    421 }
    422 
    423 status_t MediaSync::setPlaybackSettings(const AudioPlaybackRate &rate) {
    424     Mutex::Autolock lock(mMutex);
    425 
    426     status_t err = setPlaybackSettings_l(rate);
    427     if (err == OK) {
    428         // TODO: adjust rate if using VSYNC as source
    429         updatePlaybackRate_l(rate.mSpeed);
    430     }
    431     return err;
    432 }
    433 
    434 status_t MediaSync::setPlaybackSettings_l(const AudioPlaybackRate &rate) {
    435     if (rate.mSpeed < 0.f || rate.mPitch < 0.f) {
    436         // We don't validate other audio settings.
    437         // They will be validated when/if audiotrack is set.
    438         return BAD_VALUE;
    439     }
    440 
    441     if (mAudioTrack != NULL) {
    442         if (rate.mSpeed == 0.f) {
    443             mAudioTrack->pause();
    444         } else {
    445             status_t err = mAudioTrack->setPlaybackRate(rate);
    446             if (err != OK) {
    447                 return BAD_VALUE;
    448             }
    449 
    450             // ignore errors
    451             (void)mAudioTrack->start();
    452         }
    453     }
    454     mPlaybackSettings = rate;
    455     return OK;
    456 }
    457 
    458 void MediaSync::getPlaybackSettings(AudioPlaybackRate *rate) {
    459     Mutex::Autolock lock(mMutex);
    460     *rate = mPlaybackSettings;
    461 }
    462 
    463 int64_t MediaSync::getRealTime(int64_t mediaTimeUs, int64_t nowUs) {
    464     int64_t realUs;
    465     if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
    466         // If failed to get current position, e.g. due to audio clock is
    467         // not ready, then just play out video immediately without delay.
    468         return nowUs;
    469     }
    470     return realUs;
    471 }
    472 
    473 int64_t MediaSync::getDurationIfPlayedAtNativeSampleRate_l(int64_t numFrames) {
    474     return (numFrames * 1000000LL / mNativeSampleRateInHz);
    475 }
    476 
    477 int64_t MediaSync::getPlayedOutAudioDurationMedia_l(int64_t nowUs) {
    478     CHECK(mAudioTrack != NULL);
    479 
    480     uint32_t numFramesPlayed;
    481     int64_t numFramesPlayedAt;
    482     AudioTimestamp ts;
    483     static const int64_t kStaleTimestamp100ms = 100000;
    484 
    485     status_t res = mAudioTrack->getTimestamp(ts);
    486     if (res == OK) {
    487         // case 1: mixing audio tracks.
    488         numFramesPlayed = ts.mPosition;
    489         numFramesPlayedAt =
    490             ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
    491         const int64_t timestampAge = nowUs - numFramesPlayedAt;
    492         if (timestampAge > kStaleTimestamp100ms) {
    493             // This is an audio FIXME.
    494             // getTimestamp returns a timestamp which may come from audio
    495             // mixing threads. After pausing, the MixerThread may go idle,
    496             // thus the mTime estimate may become stale. Assuming that the
    497             // MixerThread runs 20ms, with FastMixer at 5ms, the max latency
    498             // should be about 25ms with an average around 12ms (to be
    499             // verified). For safety we use 100ms.
    500             ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) "
    501                   "numFramesPlayedAt(%lld)",
    502                   (long long)nowUs, (long long)numFramesPlayedAt);
    503             numFramesPlayedAt = nowUs - kStaleTimestamp100ms;
    504         }
    505         //ALOGD("getTimestamp: OK %d %lld",
    506         //      numFramesPlayed, (long long)numFramesPlayedAt);
    507     } else if (res == WOULD_BLOCK) {
    508         // case 2: transitory state on start of a new track
    509         numFramesPlayed = 0;
    510         numFramesPlayedAt = nowUs;
    511         //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
    512         //      numFramesPlayed, (long long)numFramesPlayedAt);
    513     } else {
    514         // case 3: transitory at new track or audio fast tracks.
    515         res = mAudioTrack->getPosition(&numFramesPlayed);
    516         CHECK_EQ(res, (status_t)OK);
    517         numFramesPlayedAt = nowUs;
    518         numFramesPlayedAt += 1000LL * mAudioTrack->latency() / 2; /* XXX */
    519         //ALOGD("getPosition: %d %lld", numFramesPlayed, (long long)numFramesPlayedAt);
    520     }
    521 
    522     //can't be negative until 12.4 hrs, test.
    523     //CHECK_EQ(numFramesPlayed & (1 << 31), 0);
    524     int64_t durationUs =
    525         getDurationIfPlayedAtNativeSampleRate_l(numFramesPlayed)
    526             + nowUs - numFramesPlayedAt;
    527     if (durationUs < 0) {
    528         // Occurs when numFramesPlayed position is very small and the following:
    529         // (1) In case 1, the time nowUs is computed before getTimestamp() is
    530         //     called and numFramesPlayedAt is greater than nowUs by time more
    531         //     than numFramesPlayed.
    532         // (2) In case 3, using getPosition and adding mAudioTrack->latency()
    533         //     to numFramesPlayedAt, by a time amount greater than
    534         //     numFramesPlayed.
    535         //
    536         // Both of these are transitory conditions.
    537         ALOGV("getPlayedOutAudioDurationMedia_l: negative duration %lld "
    538               "set to zero", (long long)durationUs);
    539         durationUs = 0;
    540     }
    541     ALOGV("getPlayedOutAudioDurationMedia_l(%lld) nowUs(%lld) frames(%u) "
    542           "framesAt(%lld)",
    543           (long long)durationUs, (long long)nowUs, numFramesPlayed,
    544           (long long)numFramesPlayedAt);
    545     return durationUs;
    546 }
    547 
    548 void MediaSync::onDrainVideo_l() {
    549     if (!isPlaying()) {
    550         return;
    551     }
    552 
    553     while (!mBufferItems.empty()) {
    554         int64_t nowUs = ALooper::GetNowUs();
    555         BufferItem *bufferItem = &*mBufferItems.begin();
    556         int64_t itemMediaUs = bufferItem->mTimestamp / 1000;
    557         int64_t itemRealUs = getRealTime(itemMediaUs, nowUs);
    558 
    559         // adjust video frame PTS based on vsync
    560         itemRealUs = mFrameScheduler->schedule(itemRealUs * 1000) / 1000;
    561         int64_t twoVsyncsUs = 2 * (mFrameScheduler->getVsyncPeriod() / 1000);
    562 
    563         // post 2 display refreshes before rendering is due
    564         if (itemRealUs <= nowUs + twoVsyncsUs) {
    565             ALOGV("adjusting PTS from %lld to %lld",
    566                     (long long)bufferItem->mTimestamp / 1000, (long long)itemRealUs);
    567             bufferItem->mTimestamp = itemRealUs * 1000;
    568             bufferItem->mIsAutoTimestamp = false;
    569 
    570             if (mHasAudio) {
    571                 if (nowUs - itemRealUs <= kMaxAllowedVideoLateTimeUs) {
    572                     renderOneBufferItem_l(*bufferItem);
    573                 } else {
    574                     // too late.
    575                     returnBufferToInput_l(
    576                             bufferItem->mGraphicBuffer, bufferItem->mFence);
    577                     mFrameScheduler->restart();
    578                 }
    579             } else {
    580                 // always render video buffer in video-only mode.
    581                 renderOneBufferItem_l(*bufferItem);
    582 
    583                 // smooth out videos >= 10fps
    584                 mMediaClock->updateAnchor(
    585                         itemMediaUs, nowUs, itemMediaUs + 100000);
    586             }
    587 
    588             mBufferItems.erase(mBufferItems.begin());
    589             mNextBufferItemMediaUs = -1;
    590         } else {
    591             if (mNextBufferItemMediaUs == -1
    592                     || mNextBufferItemMediaUs > itemMediaUs) {
    593                 sp<AMessage> msg = new AMessage(kWhatDrainVideo, this);
    594                 msg->post(itemRealUs - nowUs - twoVsyncsUs);
    595                 mNextBufferItemMediaUs = itemMediaUs;
    596             }
    597             break;
    598         }
    599     }
    600 }
    601 
    602 void MediaSync::onFrameAvailableFromInput() {
    603     Mutex::Autolock lock(mMutex);
    604 
    605     const static nsecs_t kAcquireWaitTimeout = 2000000000; // 2 seconds
    606 
    607     mReturnPendingInputFrame = false;
    608 
    609     // If there are too many outstanding buffers, wait until a buffer is
    610     // released back to the input in onBufferReleased.
    611     // NOTE: BufferQueue allows dequeuing maxAcquiredBufferCount + 1 buffers
    612     while (mNumOutstandingBuffers > mMaxAcquiredBufferCount
    613             && !mIsAbandoned && !mReturnPendingInputFrame) {
    614         if (mReleaseCondition.waitRelative(mMutex, kAcquireWaitTimeout) != OK) {
    615             ALOGI_IF(mPlaybackRate != 0.f, "still waiting to release a buffer before acquire");
    616         }
    617 
    618         // If the sync is abandoned while we are waiting, the release
    619         // condition variable will be broadcast, and we should just return
    620         // without attempting to do anything more (since the input queue will
    621         // also be abandoned).
    622         if (mIsAbandoned) {
    623             return;
    624         }
    625     }
    626 
    627     // Acquire and detach the buffer from the input.
    628     BufferItem bufferItem;
    629     status_t status = mInput->acquireBuffer(&bufferItem, 0 /* presentWhen */);
    630     if (status != NO_ERROR) {
    631         ALOGE("acquiring buffer from input failed (%d)", status);
    632         return;
    633     }
    634     ++mNumOutstandingBuffers;
    635 
    636     ALOGV("acquired buffer %#llx from input", (long long)bufferItem.mGraphicBuffer->getId());
    637 
    638     status = mInput->detachBuffer(bufferItem.mSlot);
    639     if (status != NO_ERROR) {
    640         ALOGE("detaching buffer from input failed (%d)", status);
    641         if (status == NO_INIT) {
    642             // If the input has been abandoned, move on.
    643             onAbandoned_l(true /* isInput */);
    644         }
    645         return;
    646     }
    647 
    648     if (mBuffersFromInput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
    649         // Something is wrong since this buffer should be at our hands, bail.
    650         ALOGE("received buffer multiple times from input");
    651         mInput->consumerDisconnect();
    652         onAbandoned_l(true /* isInput */);
    653         return;
    654     }
    655     mBuffersFromInput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
    656 
    657     // If flush happened while waiting for a buffer to be released, simply return it
    658     // TRICKY: do it here after it is detached so that we don't have to cache mGraphicBuffer.
    659     if (mReturnPendingInputFrame) {
    660         mReturnPendingInputFrame = false;
    661         returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence);
    662         return;
    663     }
    664 
    665     mBufferItems.push_back(bufferItem);
    666 
    667     if (mBufferItems.size() == 1) {
    668         onDrainVideo_l();
    669     }
    670 }
    671 
    672 void MediaSync::renderOneBufferItem_l(const BufferItem &bufferItem) {
    673     IGraphicBufferProducer::QueueBufferInput queueInput(
    674             bufferItem.mTimestamp,
    675             bufferItem.mIsAutoTimestamp,
    676             bufferItem.mDataSpace,
    677             bufferItem.mCrop,
    678             static_cast<int32_t>(bufferItem.mScalingMode),
    679             bufferItem.mTransform,
    680             bufferItem.mFence);
    681 
    682     // Attach and queue the buffer to the output.
    683     int slot;
    684     mOutput->setGenerationNumber(bufferItem.mGraphicBuffer->getGenerationNumber());
    685     status_t status = mOutput->attachBuffer(&slot, bufferItem.mGraphicBuffer);
    686     ALOGE_IF(status != NO_ERROR, "attaching buffer to output failed (%d)", status);
    687     if (status == NO_ERROR) {
    688         IGraphicBufferProducer::QueueBufferOutput queueOutput;
    689         status = mOutput->queueBuffer(slot, queueInput, &queueOutput);
    690         ALOGE_IF(status != NO_ERROR, "queueing buffer to output failed (%d)", status);
    691     }
    692 
    693     if (status != NO_ERROR) {
    694         returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence);
    695         if (status == NO_INIT) {
    696             // If the output has been abandoned, move on.
    697             onAbandoned_l(false /* isInput */);
    698         }
    699         return;
    700     }
    701 
    702     if (mBuffersSentToOutput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
    703         // Something is wrong since this buffer should be held by output now, bail.
    704         mInput->consumerDisconnect();
    705         onAbandoned_l(true /* isInput */);
    706         return;
    707     }
    708     mBuffersSentToOutput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
    709 
    710     ALOGV("queued buffer %#llx to output", (long long)bufferItem.mGraphicBuffer->getId());
    711 }
    712 
    713 void MediaSync::onBufferReleasedByOutput(sp<IGraphicBufferProducer> &output) {
    714     Mutex::Autolock lock(mMutex);
    715 
    716     if (output != mOutput) {
    717         return;  // This is not the current output, ignore.
    718     }
    719 
    720     sp<GraphicBuffer> buffer;
    721     sp<Fence> fence;
    722     status_t status = mOutput->detachNextBuffer(&buffer, &fence);
    723     ALOGE_IF(status != NO_ERROR, "detaching buffer from output failed (%d)", status);
    724 
    725     if (status == NO_INIT) {
    726         // If the output has been abandoned, we can't do anything else,
    727         // since buffer is invalid.
    728         onAbandoned_l(false /* isInput */);
    729         return;
    730     }
    731 
    732     ALOGV("detached buffer %#llx from output", (long long)buffer->getId());
    733 
    734     // If we've been abandoned, we can't return the buffer to the input, so just
    735     // move on.
    736     if (mIsAbandoned) {
    737         return;
    738     }
    739 
    740     ssize_t ix = mBuffersSentToOutput.indexOfKey(buffer->getId());
    741     if (ix < 0) {
    742         // The buffer is unknown, maybe leftover, ignore.
    743         return;
    744     }
    745     mBuffersSentToOutput.removeItemsAt(ix);
    746 
    747     returnBufferToInput_l(buffer, fence);
    748 }
    749 
    750 void MediaSync::returnBufferToInput_l(
    751         const sp<GraphicBuffer> &buffer, const sp<Fence> &fence) {
    752     ssize_t ix = mBuffersFromInput.indexOfKey(buffer->getId());
    753     if (ix < 0) {
    754         // The buffer is unknown, something is wrong, bail.
    755         ALOGE("output returned unknown buffer");
    756         mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
    757         onAbandoned_l(false /* isInput */);
    758         return;
    759     }
    760     sp<GraphicBuffer> oldBuffer = mBuffersFromInput.valueAt(ix);
    761     mBuffersFromInput.removeItemsAt(ix);
    762 
    763     // Attach and release the buffer back to the input.
    764     int consumerSlot;
    765     status_t status = mInput->attachBuffer(&consumerSlot, oldBuffer);
    766     ALOGE_IF(status != NO_ERROR, "attaching buffer to input failed (%d)", status);
    767     if (status == NO_ERROR) {
    768         status = mInput->releaseBuffer(consumerSlot, 0 /* frameNumber */,
    769                 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
    770         ALOGE_IF(status != NO_ERROR, "releasing buffer to input failed (%d)", status);
    771     }
    772 
    773     // Notify any waiting onFrameAvailable calls.
    774     --mNumOutstandingBuffers;
    775     mReleaseCondition.signal();
    776 
    777     if (status == NO_ERROR) {
    778         ALOGV("released buffer %#llx to input", (long long)oldBuffer->getId());
    779     }
    780 }
    781 
    782 void MediaSync::onAbandoned_l(bool isInput) {
    783     ALOGE("the %s has abandoned me", (isInput ? "input" : "output"));
    784     if (!mIsAbandoned) {
    785         if (isInput) {
    786             mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
    787         } else {
    788             mInput->consumerDisconnect();
    789         }
    790         mIsAbandoned = true;
    791     }
    792     mReleaseCondition.broadcast();
    793 }
    794 
    795 void MediaSync::onMessageReceived(const sp<AMessage> &msg) {
    796     switch (msg->what()) {
    797         case kWhatDrainVideo:
    798         {
    799             Mutex::Autolock lock(mMutex);
    800             if (mNextBufferItemMediaUs != -1) {
    801                 int64_t nowUs = ALooper::GetNowUs();
    802                 int64_t itemRealUs = getRealTime(mNextBufferItemMediaUs, nowUs);
    803 
    804                 // The message could arrive earlier than expected due to
    805                 // various reasons, e.g., media clock has been changed because
    806                 // of new anchor time or playback rate. In such cases, the
    807                 // message needs to be re-posted.
    808                 if (itemRealUs > nowUs) {
    809                     msg->post(itemRealUs - nowUs);
    810                     break;
    811                 }
    812             }
    813 
    814             onDrainVideo_l();
    815             break;
    816         }
    817 
    818         default:
    819             TRESPASS();
    820             break;
    821     }
    822 }
    823 
    824 MediaSync::InputListener::InputListener(const sp<MediaSync> &sync)
    825       : mSync(sync) {}
    826 
    827 MediaSync::InputListener::~InputListener() {}
    828 
    829 void MediaSync::InputListener::onFrameAvailable(const BufferItem &/* item */) {
    830     mSync->onFrameAvailableFromInput();
    831 }
    832 
    833 // We don't care about sideband streams, since we won't relay them.
    834 void MediaSync::InputListener::onSidebandStreamChanged() {
    835     ALOGE("onSidebandStreamChanged: got sideband stream unexpectedly.");
    836 }
    837 
    838 
    839 void MediaSync::InputListener::binderDied(const wp<IBinder> &/* who */) {
    840     Mutex::Autolock lock(mSync->mMutex);
    841     mSync->onAbandoned_l(true /* isInput */);
    842 }
    843 
    844 MediaSync::OutputListener::OutputListener(const sp<MediaSync> &sync,
    845         const sp<IGraphicBufferProducer> &output)
    846       : mSync(sync),
    847         mOutput(output) {}
    848 
    849 MediaSync::OutputListener::~OutputListener() {}
    850 
    851 void MediaSync::OutputListener::onBufferReleased() {
    852     mSync->onBufferReleasedByOutput(mOutput);
    853 }
    854 
    855 void MediaSync::OutputListener::binderDied(const wp<IBinder> &/* who */) {
    856     Mutex::Autolock lock(mSync->mMutex);
    857     mSync->onAbandoned_l(false /* isInput */);
    858 }
    859 
    860 } // namespace android
    861