Home | History | Annotate | Download | only in omx 
      1 /*
      2  * Copyright (C) 2013 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #include <inttypes.h>
     18 
     19 #define LOG_TAG "GraphicBufferSource"
     20 //#define LOG_NDEBUG 0
     21 #include <utils/Log.h>
     22 
     23 #include "GraphicBufferSource.h"
     24 
     25 #include <OMX_Core.h>
     26 #include <OMX_IndexExt.h>
     27 #include <media/stagefright/foundation/ADebug.h>
     28 #include <media/stagefright/foundation/AMessage.h>
     29 
     30 #include <media/hardware/MetadataBufferType.h>
     31 #include <ui/GraphicBuffer.h>
     32 #include <gui/BufferItem.h>
     33 #include <HardwareAPI.h>
     34 
     35 #include <inttypes.h>
     36 #include "FrameDropper.h"
     37 
     38 namespace android {
     39 
     40 static const bool EXTRA_CHECK = true;
     41 
     42 GraphicBufferSource::PersistentProxyListener::PersistentProxyListener(
     43         const wp<IGraphicBufferConsumer> &consumer,
     44         const wp<ConsumerListener>& consumerListener) :
     45     mConsumerListener(consumerListener),
     46     mConsumer(consumer) {}
     47 
     48 GraphicBufferSource::PersistentProxyListener::~PersistentProxyListener() {}
     49 
     50 void GraphicBufferSource::PersistentProxyListener::onFrameAvailable(
     51         const BufferItem& item) {
     52     sp<ConsumerListener> listener(mConsumerListener.promote());
     53     if (listener != NULL) {
     54         listener->onFrameAvailable(item);
     55     } else {
     56         sp<IGraphicBufferConsumer> consumer(mConsumer.promote());
     57         if (consumer == NULL) {
     58             return;
     59         }
     60         BufferItem bi;
     61         status_t err = consumer->acquireBuffer(&bi, 0);
     62         if (err != OK) {
     63             ALOGE("PersistentProxyListener: acquireBuffer failed (%d)", err);
     64             return;
     65         }
     66 
     67         err = consumer->detachBuffer(bi.mBuf);
     68         if (err != OK) {
     69             ALOGE("PersistentProxyListener: detachBuffer failed (%d)", err);
     70             return;
     71         }
     72 
     73         err = consumer->attachBuffer(&bi.mBuf, bi.mGraphicBuffer);
     74         if (err != OK) {
     75             ALOGE("PersistentProxyListener: attachBuffer failed (%d)", err);
     76             return;
     77         }
     78 
     79         err = consumer->releaseBuffer(bi.mBuf, 0,
     80                 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, bi.mFence);
     81         if (err != OK) {
     82             ALOGE("PersistentProxyListener: releaseBuffer failed (%d)", err);
     83         }
     84     }
     85 }
     86 
     87 void GraphicBufferSource::PersistentProxyListener::onFrameReplaced(
     88         const BufferItem& item) {
     89     sp<ConsumerListener> listener(mConsumerListener.promote());
     90     if (listener != NULL) {
     91         listener->onFrameReplaced(item);
     92     }
     93 }
     94 
     95 void GraphicBufferSource::PersistentProxyListener::onBuffersReleased() {
     96     sp<ConsumerListener> listener(mConsumerListener.promote());
     97     if (listener != NULL) {
     98         listener->onBuffersReleased();
     99     }
    100 }
    101 
    102 void GraphicBufferSource::PersistentProxyListener::onSidebandStreamChanged() {
    103     sp<ConsumerListener> listener(mConsumerListener.promote());
    104     if (listener != NULL) {
    105         listener->onSidebandStreamChanged();
    106     }
    107 }
    108 
    109 GraphicBufferSource::GraphicBufferSource(
    110         OMXNodeInstance* nodeInstance,
    111         uint32_t bufferWidth,
    112         uint32_t bufferHeight,
    113         uint32_t bufferCount,
    114         uint32_t consumerUsage,
    115         const sp<IGraphicBufferConsumer> &consumer) :
    116     mInitCheck(UNKNOWN_ERROR),
    117     mNodeInstance(nodeInstance),
    118     mExecuting(false),
    119     mSuspended(false),
    120     mIsPersistent(false),
    121     mConsumer(consumer),
    122     mNumFramesAvailable(0),
    123     mNumBufferAcquired(0),
    124     mEndOfStream(false),
    125     mEndOfStreamSent(false),
    126     mMaxTimestampGapUs(-1ll),
    127     mPrevOriginalTimeUs(-1ll),
    128     mPrevModifiedTimeUs(-1ll),
    129     mSkipFramesBeforeNs(-1ll),
    130     mRepeatAfterUs(-1ll),
    131     mRepeatLastFrameGeneration(0),
    132     mRepeatLastFrameTimestamp(-1ll),
    133     mLatestBufferId(-1),
    134     mLatestBufferFrameNum(0),
    135     mLatestBufferUseCount(0),
    136     mLatestBufferFence(Fence::NO_FENCE),
    137     mRepeatBufferDeferred(false),
    138     mTimePerCaptureUs(-1ll),
    139     mTimePerFrameUs(-1ll),
    140     mPrevCaptureUs(-1ll),
    141     mPrevFrameUs(-1ll) {
    142 
    143     ALOGV("GraphicBufferSource w=%u h=%u c=%u",
    144             bufferWidth, bufferHeight, bufferCount);
    145 
    146     if (bufferWidth == 0 || bufferHeight == 0) {
    147         ALOGE("Invalid dimensions %ux%u", bufferWidth, bufferHeight);
    148         mInitCheck = BAD_VALUE;
    149         return;
    150     }
    151 
    152     if (mConsumer == NULL) {
    153         String8 name("GraphicBufferSource");
    154 
    155         BufferQueue::createBufferQueue(&mProducer, &mConsumer);
    156         mConsumer->setConsumerName(name);
    157 
    158         // use consumer usage bits queried from encoder, but always add HW_VIDEO_ENCODER
    159         // for backward compatibility.
    160         consumerUsage |= GRALLOC_USAGE_HW_VIDEO_ENCODER;
    161         mConsumer->setConsumerUsageBits(consumerUsage);
    162 
    163         mInitCheck = mConsumer->setMaxAcquiredBufferCount(bufferCount);
    164         if (mInitCheck != NO_ERROR) {
    165             ALOGE("Unable to set BQ max acquired buffer count to %u: %d",
    166                     bufferCount, mInitCheck);
    167             return;
    168         }
    169     } else {
    170         mIsPersistent = true;
    171     }
    172     mConsumer->setDefaultBufferSize(bufferWidth, bufferHeight);
    173     // Note that we can't create an sp<...>(this) in a ctor that will not keep a
    174     // reference once the ctor ends, as that would cause the refcount of 'this'
    175     // dropping to 0 at the end of the ctor.  Since all we need is a wp<...>
    176     // that's what we create.
    177     wp<BufferQueue::ConsumerListener> listener = static_cast<BufferQueue::ConsumerListener*>(this);
    178     sp<IConsumerListener> proxy;
    179     if (!mIsPersistent) {
    180         proxy = new BufferQueue::ProxyConsumerListener(listener);
    181     } else {
    182         proxy = new PersistentProxyListener(mConsumer, listener);
    183     }
    184 
    185     mInitCheck = mConsumer->consumerConnect(proxy, false);
    186     if (mInitCheck != NO_ERROR) {
    187         ALOGE("Error connecting to BufferQueue: %s (%d)",
    188                 strerror(-mInitCheck), mInitCheck);
    189         return;
    190     }
    191 
    192     CHECK(mInitCheck == NO_ERROR);
    193 }
    194 
    195 GraphicBufferSource::~GraphicBufferSource() {
    196     if (mLatestBufferId >= 0) {
    197         releaseBuffer(
    198                 mLatestBufferId, mLatestBufferFrameNum,
    199                 mBufferSlot[mLatestBufferId], mLatestBufferFence);
    200     }
    201     if (mNumBufferAcquired != 0) {
    202         ALOGW("potential buffer leak (acquired %d)", mNumBufferAcquired);
    203     }
    204     if (mConsumer != NULL && !mIsPersistent) {
    205         status_t err = mConsumer->consumerDisconnect();
    206         if (err != NO_ERROR) {
    207             ALOGW("consumerDisconnect failed: %d", err);
    208         }
    209     }
    210 }
    211 
    212 void GraphicBufferSource::omxExecuting() {
    213     Mutex::Autolock autoLock(mMutex);
    214     ALOGV("--> executing; avail=%zu, codec vec size=%zd",
    215             mNumFramesAvailable, mCodecBuffers.size());
    216     CHECK(!mExecuting);
    217     mExecuting = true;
    218 
    219     // Start by loading up as many buffers as possible.  We want to do this,
    220     // rather than just submit the first buffer, to avoid a degenerate case:
    221     // if all BQ buffers arrive before we start executing, and we only submit
    222     // one here, the other BQ buffers will just sit until we get notified
    223     // that the codec buffer has been released.  We'd then acquire and
    224     // submit a single additional buffer, repeatedly, never using more than
    225     // one codec buffer simultaneously.  (We could instead try to submit
    226     // all BQ buffers whenever any codec buffer is freed, but if we get the
    227     // initial conditions right that will never be useful.)
    228     while (mNumFramesAvailable) {
    229         if (!fillCodecBuffer_l()) {
    230             ALOGV("stop load with frames available (codecAvail=%d)",
    231                     isCodecBufferAvailable_l());
    232             break;
    233         }
    234     }
    235 
    236     ALOGV("done loading initial frames, avail=%zu", mNumFramesAvailable);
    237 
    238     // If EOS has already been signaled, and there are no more frames to
    239     // submit, try to send EOS now as well.
    240     if (mEndOfStream && mNumFramesAvailable == 0) {
    241         submitEndOfInputStream_l();
    242     }
    243 
    244     if (mRepeatAfterUs > 0ll && mLooper == NULL) {
    245         mReflector = new AHandlerReflector<GraphicBufferSource>(this);
    246 
    247         mLooper = new ALooper;
    248         mLooper->registerHandler(mReflector);
    249         mLooper->start();
    250 
    251         if (mLatestBufferId >= 0) {
    252             sp<AMessage> msg =
    253                 new AMessage(kWhatRepeatLastFrame, mReflector);
    254 
    255             msg->setInt32("generation", ++mRepeatLastFrameGeneration);
    256             msg->post(mRepeatAfterUs);
    257         }
    258     }
    259 }
    260 
    261 void GraphicBufferSource::omxIdle() {
    262     ALOGV("omxIdle");
    263 
    264     Mutex::Autolock autoLock(mMutex);
    265 
    266     if (mExecuting) {
    267         // We are only interested in the transition from executing->idle,
    268         // not loaded->idle.
    269         mExecuting = false;
    270     }
    271 }
    272 
    273 void GraphicBufferSource::omxLoaded(){
    274     Mutex::Autolock autoLock(mMutex);
    275     if (!mExecuting) {
    276         // This can happen if something failed very early.
    277         ALOGW("Dropped back down to Loaded without Executing");
    278     }
    279 
    280     if (mLooper != NULL) {
    281         mLooper->unregisterHandler(mReflector->id());
    282         mReflector.clear();
    283 
    284         mLooper->stop();
    285         mLooper.clear();
    286     }
    287 
    288     ALOGV("--> loaded; avail=%zu eos=%d eosSent=%d",
    289             mNumFramesAvailable, mEndOfStream, mEndOfStreamSent);
    290 
    291     // Codec is no longer executing.  Discard all codec-related state.
    292     mCodecBuffers.clear();
    293     // TODO: scan mCodecBuffers to verify that all mGraphicBuffer entries
    294     //       are null; complain if not
    295 
    296     mExecuting = false;
    297 }
    298 
    299 void GraphicBufferSource::addCodecBuffer(OMX_BUFFERHEADERTYPE* header) {
    300     Mutex::Autolock autoLock(mMutex);
    301 
    302     if (mExecuting) {
    303         // This should never happen -- buffers can only be allocated when
    304         // transitioning from "loaded" to "idle".
    305         ALOGE("addCodecBuffer: buffer added while executing");
    306         return;
    307     }
    308 
    309     ALOGV("addCodecBuffer h=%p size=%" PRIu32 " p=%p",
    310             header, header->nAllocLen, header->pBuffer);
    311     CodecBuffer codecBuffer;
    312     codecBuffer.mHeader = header;
    313     mCodecBuffers.add(codecBuffer);
    314 }
    315 
    316 void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header, int fenceFd) {
    317     Mutex::Autolock autoLock(mMutex);
    318     if (!mExecuting) {
    319         return;
    320     }
    321 
    322     int cbi = findMatchingCodecBuffer_l(header);
    323     if (cbi < 0) {
    324         // This should never happen.
    325         ALOGE("codecBufferEmptied: buffer not recognized (h=%p)", header);
    326         if (fenceFd >= 0) {
    327             ::close(fenceFd);
    328         }
    329         return;
    330     }
    331 
    332     ALOGV("codecBufferEmptied h=%p size=%" PRIu32 " filled=%" PRIu32 " p=%p",
    333             header, header->nAllocLen, header->nFilledLen,
    334             header->pBuffer);
    335     CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
    336 
    337     // header->nFilledLen may not be the original value, so we can't compare
    338     // that to zero to see of this was the EOS buffer.  Instead we just
    339     // see if the GraphicBuffer reference was null, which should only ever
    340     // happen for EOS.
    341     if (codecBuffer.mGraphicBuffer == NULL) {
    342         if (!(mEndOfStream && mEndOfStreamSent)) {
    343             // This can happen when broken code sends us the same buffer
    344             // twice in a row.
    345             ALOGE("ERROR: codecBufferEmptied on non-EOS null buffer "
    346                     "(buffer emptied twice?)");
    347         }
    348         // No GraphicBuffer to deal with, no additional input or output is
    349         // expected, so just return.
    350         if (fenceFd >= 0) {
    351             ::close(fenceFd);
    352         }
    353         return;
    354     }
    355 
    356     if (EXTRA_CHECK && header->nAllocLen >= sizeof(MetadataBufferType)) {
    357         // Pull the graphic buffer handle back out of the buffer, and confirm
    358         // that it matches expectations.
    359         OMX_U8* data = header->pBuffer;
    360         MetadataBufferType type = *(MetadataBufferType *)data;
    361         if (type == kMetadataBufferTypeGrallocSource
    362                 && header->nAllocLen >= sizeof(VideoGrallocMetadata)) {
    363             VideoGrallocMetadata &grallocMeta = *(VideoGrallocMetadata *)data;
    364             if (grallocMeta.pHandle != codecBuffer.mGraphicBuffer->handle) {
    365                 // should never happen
    366                 ALOGE("codecBufferEmptied: buffer's handle is %p, expected %p",
    367                         grallocMeta.pHandle, codecBuffer.mGraphicBuffer->handle);
    368                 CHECK(!"codecBufferEmptied: mismatched buffer");
    369             }
    370         } else if (type == kMetadataBufferTypeANWBuffer
    371                 && header->nAllocLen >= sizeof(VideoNativeMetadata)) {
    372             VideoNativeMetadata &nativeMeta = *(VideoNativeMetadata *)data;
    373             if (nativeMeta.pBuffer != codecBuffer.mGraphicBuffer->getNativeBuffer()) {
    374                 // should never happen
    375                 ALOGE("codecBufferEmptied: buffer is %p, expected %p",
    376                         nativeMeta.pBuffer, codecBuffer.mGraphicBuffer->getNativeBuffer());
    377                 CHECK(!"codecBufferEmptied: mismatched buffer");
    378             }
    379         }
    380     }
    381 
    382     // Find matching entry in our cached copy of the BufferQueue slots.
    383     // If we find a match, release that slot.  If we don't, the BufferQueue
    384     // has dropped that GraphicBuffer, and there's nothing for us to release.
    385     int id = codecBuffer.mBuf;
    386     sp<Fence> fence = new Fence(fenceFd);
    387     if (mBufferSlot[id] != NULL &&
    388         mBufferSlot[id]->handle == codecBuffer.mGraphicBuffer->handle) {
    389         ALOGV("cbi %d matches bq slot %d, handle=%p",
    390                 cbi, id, mBufferSlot[id]->handle);
    391 
    392         if (id == mLatestBufferId) {
    393             CHECK_GT(mLatestBufferUseCount--, 0);
    394         } else {
    395             releaseBuffer(id, codecBuffer.mFrameNumber, mBufferSlot[id], fence);
    396         }
    397     } else {
    398         ALOGV("codecBufferEmptied: no match for emptied buffer in cbi %d",
    399                 cbi);
    400         // we will not reuse codec buffer, so there is no need to wait for fence
    401     }
    402 
    403     // Mark the codec buffer as available by clearing the GraphicBuffer ref.
    404     codecBuffer.mGraphicBuffer = NULL;
    405 
    406     if (mNumFramesAvailable) {
    407         // Fill this codec buffer.
    408         CHECK(!mEndOfStreamSent);
    409         ALOGV("buffer freed, %zu frames avail (eos=%d)",
    410                 mNumFramesAvailable, mEndOfStream);
    411         fillCodecBuffer_l();
    412     } else if (mEndOfStream) {
    413         // No frames available, but EOS is pending, so use this buffer to
    414         // send that.
    415         ALOGV("buffer freed, EOS pending");
    416         submitEndOfInputStream_l();
    417     } else if (mRepeatBufferDeferred) {
    418         bool success = repeatLatestBuffer_l();
    419         if (success) {
    420             ALOGV("deferred repeatLatestBuffer_l SUCCESS");
    421         } else {
    422             ALOGV("deferred repeatLatestBuffer_l FAILURE");
    423         }
    424         mRepeatBufferDeferred = false;
    425     }
    426 
    427     return;
    428 }
    429 
    430 void GraphicBufferSource::codecBufferFilled(OMX_BUFFERHEADERTYPE* header) {
    431     Mutex::Autolock autoLock(mMutex);
    432 
    433     if (mMaxTimestampGapUs > 0ll
    434             && !(header->nFlags & OMX_BUFFERFLAG_CODECCONFIG)) {
    435         ssize_t index = mOriginalTimeUs.indexOfKey(header->nTimeStamp);
    436         if (index >= 0) {
    437             ALOGV("OUT timestamp: %lld -> %lld",
    438                     static_cast<long long>(header->nTimeStamp),
    439                     static_cast<long long>(mOriginalTimeUs[index]));
    440             header->nTimeStamp = mOriginalTimeUs[index];
    441             mOriginalTimeUs.removeItemsAt(index);
    442         } else {
    443             // giving up the effort as encoder doesn't appear to preserve pts
    444             ALOGW("giving up limiting timestamp gap (pts = %lld)",
    445                     header->nTimeStamp);
    446             mMaxTimestampGapUs = -1ll;
    447         }
    448         if (mOriginalTimeUs.size() > BufferQueue::NUM_BUFFER_SLOTS) {
    449             // something terribly wrong must have happened, giving up...
    450             ALOGE("mOriginalTimeUs has too many entries (%zu)",
    451                     mOriginalTimeUs.size());
    452             mMaxTimestampGapUs = -1ll;
    453         }
    454     }
    455 }
    456 
    457 void GraphicBufferSource::suspend(bool suspend) {
    458     Mutex::Autolock autoLock(mMutex);
    459 
    460     if (suspend) {
    461         mSuspended = true;
    462 
    463         while (mNumFramesAvailable > 0) {
    464             BufferItem item;
    465             status_t err = mConsumer->acquireBuffer(&item, 0);
    466 
    467             if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
    468                 // shouldn't happen.
    469                 ALOGW("suspend: frame was not available");
    470                 break;
    471             } else if (err != OK) {
    472                 ALOGW("suspend: acquireBuffer returned err=%d", err);
    473                 break;
    474             }
    475 
    476             ++mNumBufferAcquired;
    477             --mNumFramesAvailable;
    478 
    479             releaseBuffer(item.mBuf, item.mFrameNumber,
    480                     item.mGraphicBuffer, item.mFence);
    481         }
    482         return;
    483     }
    484 
    485     mSuspended = false;
    486 
    487     if (mExecuting && mNumFramesAvailable == 0 && mRepeatBufferDeferred) {
    488         if (repeatLatestBuffer_l()) {
    489             ALOGV("suspend/deferred repeatLatestBuffer_l SUCCESS");
    490 
    491             mRepeatBufferDeferred = false;
    492         } else {
    493             ALOGV("suspend/deferred repeatLatestBuffer_l FAILURE");
    494         }
    495     }
    496 }
    497 
    498 bool GraphicBufferSource::fillCodecBuffer_l() {
    499     CHECK(mExecuting && mNumFramesAvailable > 0);
    500 
    501     if (mSuspended) {
    502         return false;
    503     }
    504 
    505     int cbi = findAvailableCodecBuffer_l();
    506     if (cbi < 0) {
    507         // No buffers available, bail.
    508         ALOGV("fillCodecBuffer_l: no codec buffers, avail now %zu",
    509                 mNumFramesAvailable);
    510         return false;
    511     }
    512 
    513     ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%zu",
    514             mNumFramesAvailable);
    515     BufferItem item;
    516     status_t err = mConsumer->acquireBuffer(&item, 0);
    517     if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
    518         // shouldn't happen
    519         ALOGW("fillCodecBuffer_l: frame was not available");
    520         return false;
    521     } else if (err != OK) {
    522         // now what? fake end-of-stream?
    523         ALOGW("fillCodecBuffer_l: acquireBuffer returned err=%d", err);
    524         return false;
    525     }
    526 
    527     mNumBufferAcquired++;
    528     mNumFramesAvailable--;
    529 
    530     // If this is the first time we're seeing this buffer, add it to our
    531     // slot table.
    532     if (item.mGraphicBuffer != NULL) {
    533         ALOGV("fillCodecBuffer_l: setting mBufferSlot %d", item.mBuf);
    534         mBufferSlot[item.mBuf] = item.mGraphicBuffer;
    535     }
    536 
    537     err = UNKNOWN_ERROR;
    538 
    539     // only submit sample if start time is unspecified, or sample
    540     // is queued after the specified start time
    541     bool dropped = false;
    542     if (mSkipFramesBeforeNs < 0ll || item.mTimestamp >= mSkipFramesBeforeNs) {
    543         // if start time is set, offset time stamp by start time
    544         if (mSkipFramesBeforeNs > 0) {
    545             item.mTimestamp -= mSkipFramesBeforeNs;
    546         }
    547 
    548         int64_t timeUs = item.mTimestamp / 1000;
    549         if (mFrameDropper != NULL && mFrameDropper->shouldDrop(timeUs)) {
    550             ALOGV("skipping frame (%lld) to meet max framerate", static_cast<long long>(timeUs));
    551             // set err to OK so that the skipped frame can still be saved as the lastest frame
    552             err = OK;
    553             dropped = true;
    554         } else {
    555             err = submitBuffer_l(item, cbi);
    556         }
    557     }
    558 
    559     if (err != OK) {
    560         ALOGV("submitBuffer_l failed, releasing bq buf %d", item.mBuf);
    561         releaseBuffer(item.mBuf, item.mFrameNumber, item.mGraphicBuffer, item.mFence);
    562     } else {
    563         ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi);
    564         setLatestBuffer_l(item, dropped);
    565     }
    566 
    567     return true;
    568 }
    569 
    570 bool GraphicBufferSource::repeatLatestBuffer_l() {
    571     CHECK(mExecuting && mNumFramesAvailable == 0);
    572 
    573     if (mLatestBufferId < 0 || mSuspended) {
    574         return false;
    575     }
    576     if (mBufferSlot[mLatestBufferId] == NULL) {
    577         // This can happen if the remote side disconnects, causing
    578         // onBuffersReleased() to NULL out our copy of the slots.  The
    579         // buffer is gone, so we have nothing to show.
    580         //
    581         // To be on the safe side we try to release the buffer.
    582         ALOGD("repeatLatestBuffer_l: slot was NULL");
    583         mConsumer->releaseBuffer(
    584                 mLatestBufferId,
    585                 mLatestBufferFrameNum,
    586                 EGL_NO_DISPLAY,
    587                 EGL_NO_SYNC_KHR,
    588                 mLatestBufferFence);
    589         mLatestBufferId = -1;
    590         mLatestBufferFrameNum = 0;
    591         mLatestBufferFence = Fence::NO_FENCE;
    592         return false;
    593     }
    594 
    595     int cbi = findAvailableCodecBuffer_l();
    596     if (cbi < 0) {
    597         // No buffers available, bail.
    598         ALOGV("repeatLatestBuffer_l: no codec buffers.");
    599         return false;
    600     }
    601 
    602     BufferItem item;
    603     item.mBuf = mLatestBufferId;
    604     item.mFrameNumber = mLatestBufferFrameNum;
    605     item.mTimestamp = mRepeatLastFrameTimestamp;
    606     item.mFence = mLatestBufferFence;
    607 
    608     status_t err = submitBuffer_l(item, cbi);
    609 
    610     if (err != OK) {
    611         return false;
    612     }
    613 
    614     ++mLatestBufferUseCount;
    615 
    616     /* repeat last frame up to kRepeatLastFrameCount times.
    617      * in case of static scene, a single repeat might not get rid of encoder
    618      * ghosting completely, refresh a couple more times to get better quality
    619      */
    620     if (--mRepeatLastFrameCount > 0) {
    621         mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
    622 
    623         if (mReflector != NULL) {
    624             sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector);
    625             msg->setInt32("generation", ++mRepeatLastFrameGeneration);
    626             msg->post(mRepeatAfterUs);
    627         }
    628     }
    629 
    630     return true;
    631 }
    632 
    633 void GraphicBufferSource::setLatestBuffer_l(
    634         const BufferItem &item, bool dropped) {
    635     ALOGV("setLatestBuffer_l");
    636 
    637     if (mLatestBufferId >= 0) {
    638         if (mLatestBufferUseCount == 0) {
    639             releaseBuffer(mLatestBufferId, mLatestBufferFrameNum,
    640                     mBufferSlot[mLatestBufferId], mLatestBufferFence);
    641             // mLatestBufferFence will be set to new fence just below
    642         }
    643     }
    644 
    645     mLatestBufferId = item.mBuf;
    646     mLatestBufferFrameNum = item.mFrameNumber;
    647     mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
    648 
    649     mLatestBufferUseCount = dropped ? 0 : 1;
    650     mRepeatBufferDeferred = false;
    651     mRepeatLastFrameCount = kRepeatLastFrameCount;
    652     mLatestBufferFence = item.mFence;
    653 
    654     if (mReflector != NULL) {
    655         sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector);
    656         msg->setInt32("generation", ++mRepeatLastFrameGeneration);
    657         msg->post(mRepeatAfterUs);
    658     }
    659 }
    660 
    661 status_t GraphicBufferSource::signalEndOfInputStream() {
    662     Mutex::Autolock autoLock(mMutex);
    663     ALOGV("signalEndOfInputStream: exec=%d avail=%zu eos=%d",
    664             mExecuting, mNumFramesAvailable, mEndOfStream);
    665 
    666     if (mEndOfStream) {
    667         ALOGE("EOS was already signaled");
    668         return INVALID_OPERATION;
    669     }
    670 
    671     // Set the end-of-stream flag.  If no frames are pending from the
    672     // BufferQueue, and a codec buffer is available, and we're executing,
    673     // we initiate the EOS from here.  Otherwise, we'll let
    674     // codecBufferEmptied() (or omxExecuting) do it.
    675     //
    676     // Note: if there are no pending frames and all codec buffers are
    677     // available, we *must* submit the EOS from here or we'll just
    678     // stall since no future events are expected.
    679     mEndOfStream = true;
    680 
    681     if (mExecuting && mNumFramesAvailable == 0) {
    682         submitEndOfInputStream_l();
    683     }
    684 
    685     return OK;
    686 }
    687 
    688 int64_t GraphicBufferSource::getTimestamp(const BufferItem &item) {
    689     int64_t timeUs = item.mTimestamp / 1000;
    690 
    691     if (mTimePerCaptureUs > 0ll) {
    692         // Time lapse or slow motion mode
    693         if (mPrevCaptureUs < 0ll) {
    694             // first capture
    695             mPrevCaptureUs = timeUs;
    696             mPrevFrameUs = timeUs;
    697         } else {
    698             // snap to nearest capture point
    699             int64_t nFrames = (timeUs + mTimePerCaptureUs / 2 - mPrevCaptureUs)
    700                     / mTimePerCaptureUs;
    701             if (nFrames <= 0) {
    702                 // skip this frame as it's too close to previous capture
    703                 ALOGV("skipping frame, timeUs %lld", static_cast<long long>(timeUs));
    704                 return -1;
    705             }
    706             mPrevCaptureUs = mPrevCaptureUs + nFrames * mTimePerCaptureUs;
    707             mPrevFrameUs += mTimePerFrameUs * nFrames;
    708         }
    709 
    710         ALOGV("timeUs %lld, captureUs %lld, frameUs %lld",
    711                 static_cast<long long>(timeUs),
    712                 static_cast<long long>(mPrevCaptureUs),
    713                 static_cast<long long>(mPrevFrameUs));
    714 
    715         return mPrevFrameUs;
    716     } else if (mMaxTimestampGapUs > 0ll) {
    717         /* Cap timestamp gap between adjacent frames to specified max
    718          *
    719          * In the scenario of cast mirroring, encoding could be suspended for
    720          * prolonged periods. Limiting the pts gap to workaround the problem
    721          * where encoder's rate control logic produces huge frames after a
    722          * long period of suspension.
    723          */
    724 
    725         int64_t originalTimeUs = timeUs;
    726         if (mPrevOriginalTimeUs >= 0ll) {
    727             if (originalTimeUs < mPrevOriginalTimeUs) {
    728                 // Drop the frame if it's going backward in time. Bad timestamp
    729                 // could disrupt encoder's rate control completely.
    730                 ALOGW("Dropping frame that's going backward in time");
    731                 return -1;
    732             }
    733             int64_t timestampGapUs = originalTimeUs - mPrevOriginalTimeUs;
    734             timeUs = (timestampGapUs < mMaxTimestampGapUs ?
    735                     timestampGapUs : mMaxTimestampGapUs) + mPrevModifiedTimeUs;
    736         }
    737         mPrevOriginalTimeUs = originalTimeUs;
    738         mPrevModifiedTimeUs = timeUs;
    739         mOriginalTimeUs.add(timeUs, originalTimeUs);
    740         ALOGV("IN  timestamp: %lld -> %lld",
    741             static_cast<long long>(originalTimeUs),
    742             static_cast<long long>(timeUs));
    743     }
    744 
    745     return timeUs;
    746 }
    747 
    748 status_t GraphicBufferSource::submitBuffer_l(const BufferItem &item, int cbi) {
    749     ALOGV("submitBuffer_l cbi=%d", cbi);
    750 
    751     int64_t timeUs = getTimestamp(item);
    752     if (timeUs < 0ll) {
    753         return UNKNOWN_ERROR;
    754     }
    755 
    756     CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
    757     codecBuffer.mGraphicBuffer = mBufferSlot[item.mBuf];
    758     codecBuffer.mBuf = item.mBuf;
    759     codecBuffer.mFrameNumber = item.mFrameNumber;
    760 
    761     OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
    762     sp<GraphicBuffer> buffer = codecBuffer.mGraphicBuffer;
    763     status_t err = mNodeInstance->emptyGraphicBuffer(
    764             header, buffer, OMX_BUFFERFLAG_ENDOFFRAME, timeUs,
    765             item.mFence->isValid() ? item.mFence->dup() : -1);
    766     if (err != OK) {
    767         ALOGW("WARNING: emptyNativeWindowBuffer failed: 0x%x", err);
    768         codecBuffer.mGraphicBuffer = NULL;
    769         return err;
    770     }
    771 
    772     ALOGV("emptyNativeWindowBuffer succeeded, h=%p p=%p buf=%p bufhandle=%p",
    773             header, header->pBuffer, buffer->getNativeBuffer(), buffer->handle);
    774     return OK;
    775 }
    776 
    777 void GraphicBufferSource::submitEndOfInputStream_l() {
    778     CHECK(mEndOfStream);
    779     if (mEndOfStreamSent) {
    780         ALOGV("EOS already sent");
    781         return;
    782     }
    783 
    784     int cbi = findAvailableCodecBuffer_l();
    785     if (cbi < 0) {
    786         ALOGV("submitEndOfInputStream_l: no codec buffers available");
    787         return;
    788     }
    789 
    790     // We reject any additional incoming graphic buffers, so there's no need
    791     // to stick a placeholder into codecBuffer.mGraphicBuffer to mark it as
    792     // in-use.
    793     CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
    794 
    795     OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
    796     status_t err = mNodeInstance->emptyGraphicBuffer(
    797             header, NULL /* buffer */, OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_EOS,
    798             0 /* timestamp */, -1 /* fenceFd */);
    799     if (err != OK) {
    800         ALOGW("emptyDirectBuffer EOS failed: 0x%x", err);
    801     } else {
    802         ALOGV("submitEndOfInputStream_l: buffer submitted, header=%p cbi=%d",
    803                 header, cbi);
    804         mEndOfStreamSent = true;
    805     }
    806 }
    807 
    808 int GraphicBufferSource::findAvailableCodecBuffer_l() {
    809     CHECK(mCodecBuffers.size() > 0);
    810 
    811     for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
    812         if (mCodecBuffers[i].mGraphicBuffer == NULL) {
    813             return i;
    814         }
    815     }
    816     return -1;
    817 }
    818 
    819 int GraphicBufferSource::findMatchingCodecBuffer_l(
    820         const OMX_BUFFERHEADERTYPE* header) {
    821     for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
    822         if (mCodecBuffers[i].mHeader == header) {
    823             return i;
    824         }
    825     }
    826     return -1;
    827 }
    828 
    829 /*
    830  * Releases an acquired buffer back to the consumer for either persistent
    831  * or non-persistent surfaces.
    832  *
    833  * id: buffer slot to release (in persistent case the id might be changed)
    834  * frameNum: frame number of the frame being released
    835  * buffer: GraphicBuffer pointer to release (note this must not be & as we
    836  *         will clear the original mBufferSlot in persistent case)
    837  * fence: fence of the frame being released
    838  */
    839 void GraphicBufferSource::releaseBuffer(
    840         int &id, uint64_t frameNum,
    841         const sp<GraphicBuffer> buffer, const sp<Fence> &fence) {
    842     if (mIsPersistent) {
    843         mConsumer->detachBuffer(id);
    844         mBufferSlot[id] = NULL;
    845 
    846         if (mConsumer->attachBuffer(&id, buffer) == OK) {
    847             mConsumer->releaseBuffer(
    848                     id, 0, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
    849         }
    850     } else {
    851         mConsumer->releaseBuffer(
    852                 id, frameNum, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
    853     }
    854     id = -1; // invalidate id
    855     mNumBufferAcquired--;
    856 }
    857 
    858 // BufferQueue::ConsumerListener callback
    859 void GraphicBufferSource::onFrameAvailable(const BufferItem& /*item*/) {
    860     Mutex::Autolock autoLock(mMutex);
    861 
    862     ALOGV("onFrameAvailable exec=%d avail=%zu",
    863             mExecuting, mNumFramesAvailable);
    864 
    865     if (mEndOfStream || mSuspended) {
    866         if (mEndOfStream) {
    867             // This should only be possible if a new buffer was queued after
    868             // EOS was signaled, i.e. the app is misbehaving.
    869 
    870             ALOGW("onFrameAvailable: EOS is set, ignoring frame");
    871         } else {
    872             ALOGV("onFrameAvailable: suspended, ignoring frame");
    873         }
    874 
    875         BufferItem item;
    876         status_t err = mConsumer->acquireBuffer(&item, 0);
    877         if (err == OK) {
    878             mNumBufferAcquired++;
    879 
    880             // If this is the first time we're seeing this buffer, add it to our
    881             // slot table.
    882             if (item.mGraphicBuffer != NULL) {
    883                 ALOGV("onFrameAvailable: setting mBufferSlot %d", item.mBuf);
    884                 mBufferSlot[item.mBuf] = item.mGraphicBuffer;
    885             }
    886 
    887             releaseBuffer(item.mBuf, item.mFrameNumber,
    888                     item.mGraphicBuffer, item.mFence);
    889         }
    890         return;
    891     }
    892 
    893     mNumFramesAvailable++;
    894 
    895     mRepeatBufferDeferred = false;
    896     ++mRepeatLastFrameGeneration;
    897 
    898     if (mExecuting) {
    899         fillCodecBuffer_l();
    900     }
    901 }
    902 
    903 // BufferQueue::ConsumerListener callback
    904 void GraphicBufferSource::onBuffersReleased() {
    905     Mutex::Autolock lock(mMutex);
    906 
    907     uint64_t slotMask;
    908     if (mConsumer->getReleasedBuffers(&slotMask) != NO_ERROR) {
    909         ALOGW("onBuffersReleased: unable to get released buffer set");
    910         slotMask = 0xffffffffffffffffULL;
    911     }
    912 
    913     ALOGV("onBuffersReleased: 0x%016" PRIx64, slotMask);
    914 
    915     for (int i = 0; i < BufferQueue::NUM_BUFFER_SLOTS; i++) {
    916         if ((slotMask & 0x01) != 0) {
    917             mBufferSlot[i] = NULL;
    918         }
    919         slotMask >>= 1;
    920     }
    921 }
    922 
    923 // BufferQueue::ConsumerListener callback
    924 void GraphicBufferSource::onSidebandStreamChanged() {
    925     ALOG_ASSERT(false, "GraphicBufferSource can't consume sideband streams");
    926 }
    927 
    928 status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs(
    929         int64_t repeatAfterUs) {
    930     Mutex::Autolock autoLock(mMutex);
    931 
    932     if (mExecuting || repeatAfterUs <= 0ll) {
    933         return INVALID_OPERATION;
    934     }
    935 
    936     mRepeatAfterUs = repeatAfterUs;
    937 
    938     return OK;
    939 }
    940 
    941 status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) {
    942     Mutex::Autolock autoLock(mMutex);
    943 
    944     if (mExecuting || maxGapUs <= 0ll) {
    945         return INVALID_OPERATION;
    946     }
    947 
    948     mMaxTimestampGapUs = maxGapUs;
    949 
    950     return OK;
    951 }
    952 
    953 status_t GraphicBufferSource::setMaxFps(float maxFps) {
    954     Mutex::Autolock autoLock(mMutex);
    955 
    956     if (mExecuting) {
    957         return INVALID_OPERATION;
    958     }
    959 
    960     mFrameDropper = new FrameDropper();
    961     status_t err = mFrameDropper->setMaxFrameRate(maxFps);
    962     if (err != OK) {
    963         mFrameDropper.clear();
    964         return err;
    965     }
    966 
    967     return OK;
    968 }
    969 
    970 void GraphicBufferSource::setSkipFramesBeforeUs(int64_t skipFramesBeforeUs) {
    971     Mutex::Autolock autoLock(mMutex);
    972 
    973     mSkipFramesBeforeNs =
    974             (skipFramesBeforeUs > 0) ? (skipFramesBeforeUs * 1000) : -1ll;
    975 }
    976 
    977 status_t GraphicBufferSource::setTimeLapseUs(int64_t* data) {
    978     Mutex::Autolock autoLock(mMutex);
    979 
    980     if (mExecuting || data[0] <= 0ll || data[1] <= 0ll) {
    981         return INVALID_OPERATION;
    982     }
    983 
    984     mTimePerFrameUs = data[0];
    985     mTimePerCaptureUs = data[1];
    986 
    987     return OK;
    988 }
    989 
    990 void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) {
    991     switch (msg->what()) {
    992         case kWhatRepeatLastFrame:
    993         {
    994             Mutex::Autolock autoLock(mMutex);
    995 
    996             int32_t generation;
    997             CHECK(msg->findInt32("generation", &generation));
    998 
    999             if (generation != mRepeatLastFrameGeneration) {
   1000                 // stale
   1001                 break;
   1002             }
   1003 
   1004             if (!mExecuting || mNumFramesAvailable > 0) {
   1005                 break;
   1006             }
   1007 
   1008             bool success = repeatLatestBuffer_l();
   1009 
   1010             if (success) {
   1011                 ALOGV("repeatLatestBuffer_l SUCCESS");
   1012             } else {
   1013                 ALOGV("repeatLatestBuffer_l FAILURE");
   1014                 mRepeatBufferDeferred = true;
   1015             }
   1016             break;
   1017         }
   1018 
   1019         default:
   1020             TRESPASS();
   1021     }
   1022 }
   1023 
   1024 }  // namespace android
   1025