Home | History | Annotate | Download | only in surfacetexture 
      1 /*
      2  * Copyright (C) 2018 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #include <cutils/compiler.h>
     18 #include <gui/BufferQueue.h>
     19 #include <math/mat4.h>
     20 #include <system/window.h>
     21 
     22 #include <utils/Trace.h>
     23 
     24 #include "Matrix.h"
     25 #include "SurfaceTexture.h"
     26 #include "ImageConsumer.h"
     27 
     28 namespace android {
     29 
     30 // Macros for including the SurfaceTexture name in log messages
     31 #define SFT_LOGV(x, ...) ALOGV("[%s] " x, mName.string(), ##__VA_ARGS__)
     32 #define SFT_LOGD(x, ...) ALOGD("[%s] " x, mName.string(), ##__VA_ARGS__)
     33 #define SFT_LOGW(x, ...) ALOGW("[%s] " x, mName.string(), ##__VA_ARGS__)
     34 #define SFT_LOGE(x, ...) ALOGE("[%s] " x, mName.string(), ##__VA_ARGS__)
     35 
     36 static const mat4 mtxIdentity;
     37 
     38 SurfaceTexture::SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t tex,
     39                                uint32_t texTarget, bool useFenceSync, bool isControlledByApp)
     40         : ConsumerBase(bq, isControlledByApp)
     41         , mCurrentCrop(Rect::EMPTY_RECT)
     42         , mCurrentTransform(0)
     43         , mCurrentScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE)
     44         , mCurrentFence(Fence::NO_FENCE)
     45         , mCurrentTimestamp(0)
     46         , mCurrentDataSpace(HAL_DATASPACE_UNKNOWN)
     47         , mCurrentFrameNumber(0)
     48         , mDefaultWidth(1)
     49         , mDefaultHeight(1)
     50         , mFilteringEnabled(true)
     51         , mTexName(tex)
     52         , mUseFenceSync(useFenceSync)
     53         , mTexTarget(texTarget)
     54         , mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT)
     55         , mOpMode(OpMode::attachedToGL) {
     56     SFT_LOGV("SurfaceTexture");
     57 
     58     memcpy(mCurrentTransformMatrix, mtxIdentity.asArray(), sizeof(mCurrentTransformMatrix));
     59 
     60     mConsumer->setConsumerUsageBits(DEFAULT_USAGE_FLAGS);
     61 }
     62 
     63 SurfaceTexture::SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t texTarget,
     64                                bool useFenceSync, bool isControlledByApp)
     65         : ConsumerBase(bq, isControlledByApp)
     66         , mCurrentCrop(Rect::EMPTY_RECT)
     67         , mCurrentTransform(0)
     68         , mCurrentScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE)
     69         , mCurrentFence(Fence::NO_FENCE)
     70         , mCurrentTimestamp(0)
     71         , mCurrentDataSpace(HAL_DATASPACE_UNKNOWN)
     72         , mCurrentFrameNumber(0)
     73         , mDefaultWidth(1)
     74         , mDefaultHeight(1)
     75         , mFilteringEnabled(true)
     76         , mTexName(0)
     77         , mUseFenceSync(useFenceSync)
     78         , mTexTarget(texTarget)
     79         , mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT)
     80         , mOpMode(OpMode::detached) {
     81     SFT_LOGV("SurfaceTexture");
     82 
     83     memcpy(mCurrentTransformMatrix, mtxIdentity.asArray(), sizeof(mCurrentTransformMatrix));
     84 
     85     mConsumer->setConsumerUsageBits(DEFAULT_USAGE_FLAGS);
     86 }
     87 
     88 status_t SurfaceTexture::setDefaultBufferSize(uint32_t w, uint32_t h) {
     89     Mutex::Autolock lock(mMutex);
     90     if (mAbandoned) {
     91         SFT_LOGE("setDefaultBufferSize: SurfaceTexture is abandoned!");
     92         return NO_INIT;
     93     }
     94     mDefaultWidth = w;
     95     mDefaultHeight = h;
     96     return mConsumer->setDefaultBufferSize(w, h);
     97 }
     98 
     99 status_t SurfaceTexture::updateTexImage() {
    100     ATRACE_CALL();
    101     SFT_LOGV("updateTexImage");
    102     Mutex::Autolock lock(mMutex);
    103 
    104     if (mAbandoned) {
    105         SFT_LOGE("updateTexImage: SurfaceTexture is abandoned!");
    106         return NO_INIT;
    107     }
    108 
    109     return mEGLConsumer.updateTexImage(*this);
    110 }
    111 
    112 status_t SurfaceTexture::releaseTexImage() {
    113     // releaseTexImage can be invoked even when not attached to a GL context.
    114     ATRACE_CALL();
    115     SFT_LOGV("releaseTexImage");
    116     Mutex::Autolock lock(mMutex);
    117 
    118     if (mAbandoned) {
    119         SFT_LOGE("releaseTexImage: SurfaceTexture is abandoned!");
    120         return NO_INIT;
    121     }
    122 
    123     return mEGLConsumer.releaseTexImage(*this);
    124 }
    125 
    126 status_t SurfaceTexture::acquireBufferLocked(BufferItem* item, nsecs_t presentWhen,
    127                                              uint64_t maxFrameNumber) {
    128     status_t err = ConsumerBase::acquireBufferLocked(item, presentWhen, maxFrameNumber);
    129     if (err != NO_ERROR) {
    130         return err;
    131     }
    132 
    133     switch (mOpMode) {
    134         case OpMode::attachedToView:
    135             mImageConsumer.onAcquireBufferLocked(item);
    136             break;
    137         case OpMode::attachedToGL:
    138             mEGLConsumer.onAcquireBufferLocked(item, *this);
    139             break;
    140         case OpMode::detached:
    141             break;
    142     }
    143 
    144     return NO_ERROR;
    145 }
    146 
    147 status_t SurfaceTexture::releaseBufferLocked(int buf, sp<GraphicBuffer> graphicBuffer,
    148                                              EGLDisplay display, EGLSyncKHR eglFence) {
    149     // release the buffer if it hasn't already been discarded by the
    150     // BufferQueue. This can happen, for example, when the producer of this
    151     // buffer has reallocated the original buffer slot after this buffer
    152     // was acquired.
    153     status_t err = ConsumerBase::releaseBufferLocked(buf, graphicBuffer, display, eglFence);
    154     // We could be releasing an EGL/Vulkan buffer, even if not currently attached to a GL context.
    155     mImageConsumer.onReleaseBufferLocked(buf);
    156     mEGLConsumer.onReleaseBufferLocked(buf);
    157     return err;
    158 }
    159 
    160 status_t SurfaceTexture::detachFromContext() {
    161     ATRACE_CALL();
    162     SFT_LOGV("detachFromContext");
    163     Mutex::Autolock lock(mMutex);
    164 
    165     if (mAbandoned) {
    166         SFT_LOGE("detachFromContext: abandoned SurfaceTexture");
    167         return NO_INIT;
    168     }
    169 
    170     if (mOpMode != OpMode::attachedToGL) {
    171         SFT_LOGE("detachFromContext: SurfaceTexture is not attached to a GL context");
    172         return INVALID_OPERATION;
    173     }
    174 
    175     status_t err = mEGLConsumer.detachFromContext(*this);
    176     if (err == OK) {
    177         mOpMode = OpMode::detached;
    178     }
    179 
    180     return err;
    181 }
    182 
    183 status_t SurfaceTexture::attachToContext(uint32_t tex) {
    184     ATRACE_CALL();
    185     SFT_LOGV("attachToContext");
    186     Mutex::Autolock lock(mMutex);
    187 
    188     if (mAbandoned) {
    189         SFT_LOGE("attachToContext: abandoned SurfaceTexture");
    190         return NO_INIT;
    191     }
    192 
    193     if (mOpMode != OpMode::detached) {
    194         SFT_LOGE(
    195                 "attachToContext: SurfaceTexture is already attached to a "
    196                 "context");
    197         return INVALID_OPERATION;
    198     }
    199 
    200     if (mCurrentTexture != BufferQueue::INVALID_BUFFER_SLOT) {
    201         // release possible ImageConsumer cache
    202         mImageConsumer.onFreeBufferLocked(mCurrentTexture);
    203     }
    204 
    205     return mEGLConsumer.attachToContext(tex, *this);
    206 }
    207 
    208 void SurfaceTexture::attachToView() {
    209     ATRACE_CALL();
    210     Mutex::Autolock _l(mMutex);
    211     if (mAbandoned) {
    212         SFT_LOGE("attachToView: abandoned SurfaceTexture");
    213         return;
    214     }
    215     if (mOpMode == OpMode::detached) {
    216         mOpMode = OpMode::attachedToView;
    217 
    218         if (mCurrentTexture != BufferQueue::INVALID_BUFFER_SLOT) {
    219             // release possible EGLConsumer texture cache
    220             mEGLConsumer.onFreeBufferLocked(mCurrentTexture);
    221             mEGLConsumer.onAbandonLocked();
    222         }
    223     } else {
    224         SFT_LOGE("attachToView: already attached");
    225     }
    226 }
    227 
    228 void SurfaceTexture::detachFromView() {
    229     ATRACE_CALL();
    230     Mutex::Autolock _l(mMutex);
    231 
    232     if (mAbandoned) {
    233         SFT_LOGE("detachFromView: abandoned SurfaceTexture");
    234         return;
    235     }
    236 
    237     if (mOpMode == OpMode::attachedToView) {
    238         mOpMode = OpMode::detached;
    239         // Free all EglImage and VkImage before the context is destroyed.
    240         for (int i=0; i < BufferQueueDefs::NUM_BUFFER_SLOTS; i++) {
    241             mImageConsumer.onFreeBufferLocked(i);
    242         }
    243     } else {
    244         SFT_LOGE("detachFromView: not attached to View");
    245     }
    246 }
    247 
    248 uint32_t SurfaceTexture::getCurrentTextureTarget() const {
    249     return mTexTarget;
    250 }
    251 
    252 void SurfaceTexture::getTransformMatrix(float mtx[16]) {
    253     Mutex::Autolock lock(mMutex);
    254     memcpy(mtx, mCurrentTransformMatrix, sizeof(mCurrentTransformMatrix));
    255 }
    256 
    257 void SurfaceTexture::setFilteringEnabled(bool enabled) {
    258     Mutex::Autolock lock(mMutex);
    259     if (mAbandoned) {
    260         SFT_LOGE("setFilteringEnabled: SurfaceTexture is abandoned!");
    261         return;
    262     }
    263     bool needsRecompute = mFilteringEnabled != enabled;
    264     mFilteringEnabled = enabled;
    265 
    266     if (needsRecompute && mCurrentTexture == BufferQueue::INVALID_BUFFER_SLOT) {
    267         SFT_LOGD("setFilteringEnabled called with no current item");
    268     }
    269 
    270     if (needsRecompute && mCurrentTexture != BufferQueue::INVALID_BUFFER_SLOT) {
    271         computeCurrentTransformMatrixLocked();
    272     }
    273 }
    274 
    275 void SurfaceTexture::computeCurrentTransformMatrixLocked() {
    276     SFT_LOGV("computeCurrentTransformMatrixLocked");
    277     sp<GraphicBuffer> buf = (mCurrentTexture == BufferQueue::INVALID_BUFFER_SLOT)
    278                                     ? nullptr
    279                                     : mSlots[mCurrentTexture].mGraphicBuffer;
    280     if (buf == nullptr) {
    281         SFT_LOGD("computeCurrentTransformMatrixLocked: no current item");
    282     }
    283     computeTransformMatrix(mCurrentTransformMatrix, buf, mCurrentCrop, mCurrentTransform,
    284                            mFilteringEnabled);
    285 }
    286 
    287 void SurfaceTexture::computeTransformMatrix(float outTransform[16], const sp<GraphicBuffer>& buf,
    288                                             const Rect& cropRect, uint32_t transform,
    289                                             bool filtering) {
    290     // Transform matrices
    291     static const mat4 mtxFlipH(-1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1);
    292     static const mat4 mtxFlipV(1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1);
    293     static const mat4 mtxRot90(0, 1, 0, 0, -1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1);
    294 
    295     mat4 xform;
    296     if (transform & NATIVE_WINDOW_TRANSFORM_FLIP_H) {
    297         xform *= mtxFlipH;
    298     }
    299     if (transform & NATIVE_WINDOW_TRANSFORM_FLIP_V) {
    300         xform *= mtxFlipV;
    301     }
    302     if (transform & NATIVE_WINDOW_TRANSFORM_ROT_90) {
    303         xform *= mtxRot90;
    304     }
    305 
    306     if (!cropRect.isEmpty() && buf.get()) {
    307         float tx = 0.0f, ty = 0.0f, sx = 1.0f, sy = 1.0f;
    308         float bufferWidth = buf->getWidth();
    309         float bufferHeight = buf->getHeight();
    310         float shrinkAmount = 0.0f;
    311         if (filtering) {
    312             // In order to prevent bilinear sampling beyond the edge of the
    313             // crop rectangle we may need to shrink it by 2 texels in each
    314             // dimension.  Normally this would just need to take 1/2 a texel
    315             // off each end, but because the chroma channels of YUV420 images
    316             // are subsampled we may need to shrink the crop region by a whole
    317             // texel on each side.
    318             switch (buf->getPixelFormat()) {
    319                 case PIXEL_FORMAT_RGBA_8888:
    320                 case PIXEL_FORMAT_RGBX_8888:
    321                 case PIXEL_FORMAT_RGBA_FP16:
    322                 case PIXEL_FORMAT_RGBA_1010102:
    323                 case PIXEL_FORMAT_RGB_888:
    324                 case PIXEL_FORMAT_RGB_565:
    325                 case PIXEL_FORMAT_BGRA_8888:
    326                     // We know there's no subsampling of any channels, so we
    327                     // only need to shrink by a half a pixel.
    328                     shrinkAmount = 0.5;
    329                     break;
    330 
    331                 default:
    332                     // If we don't recognize the format, we must assume the
    333                     // worst case (that we care about), which is YUV420.
    334                     shrinkAmount = 1.0;
    335                     break;
    336             }
    337         }
    338 
    339         // Only shrink the dimensions that are not the size of the buffer.
    340         if (cropRect.width() < bufferWidth) {
    341             tx = (float(cropRect.left) + shrinkAmount) / bufferWidth;
    342             sx = (float(cropRect.width()) - (2.0f * shrinkAmount)) / bufferWidth;
    343         }
    344         if (cropRect.height() < bufferHeight) {
    345             ty = (float(bufferHeight - cropRect.bottom) + shrinkAmount) / bufferHeight;
    346             sy = (float(cropRect.height()) - (2.0f * shrinkAmount)) / bufferHeight;
    347         }
    348 
    349         mat4 crop(sx, 0, 0, 0, 0, sy, 0, 0, 0, 0, 1, 0, tx, ty, 0, 1);
    350         xform = crop * xform;
    351     }
    352 
    353     // SurfaceFlinger expects the top of its window textures to be at a Y
    354     // coordinate of 0, so SurfaceTexture must behave the same way.  We don't
    355     // want to expose this to applications, however, so we must add an
    356     // additional vertical flip to the transform after all the other transforms.
    357     xform = mtxFlipV * xform;
    358 
    359     memcpy(outTransform, xform.asArray(), sizeof(xform));
    360 }
    361 
    362 Rect SurfaceTexture::scaleDownCrop(const Rect& crop, uint32_t bufferWidth, uint32_t bufferHeight) {
    363     Rect outCrop = crop;
    364 
    365     uint32_t newWidth = static_cast<uint32_t>(crop.width());
    366     uint32_t newHeight = static_cast<uint32_t>(crop.height());
    367 
    368     if (newWidth * bufferHeight > newHeight * bufferWidth) {
    369         newWidth = newHeight * bufferWidth / bufferHeight;
    370         ALOGV("too wide: newWidth = %d", newWidth);
    371     } else if (newWidth * bufferHeight < newHeight * bufferWidth) {
    372         newHeight = newWidth * bufferHeight / bufferWidth;
    373         ALOGV("too tall: newHeight = %d", newHeight);
    374     }
    375 
    376     uint32_t currentWidth = static_cast<uint32_t>(crop.width());
    377     uint32_t currentHeight = static_cast<uint32_t>(crop.height());
    378 
    379     // The crop is too wide
    380     if (newWidth < currentWidth) {
    381         uint32_t dw = currentWidth - newWidth;
    382         auto halfdw = dw / 2;
    383         outCrop.left += halfdw;
    384         // Not halfdw because it would subtract 1 too few when dw is odd
    385         outCrop.right -= (dw - halfdw);
    386         // The crop is too tall
    387     } else if (newHeight < currentHeight) {
    388         uint32_t dh = currentHeight - newHeight;
    389         auto halfdh = dh / 2;
    390         outCrop.top += halfdh;
    391         // Not halfdh because it would subtract 1 too few when dh is odd
    392         outCrop.bottom -= (dh - halfdh);
    393     }
    394 
    395     ALOGV("getCurrentCrop final crop [%d,%d,%d,%d]", outCrop.left, outCrop.top, outCrop.right,
    396           outCrop.bottom);
    397 
    398     return outCrop;
    399 }
    400 
    401 nsecs_t SurfaceTexture::getTimestamp() {
    402     SFT_LOGV("getTimestamp");
    403     Mutex::Autolock lock(mMutex);
    404     return mCurrentTimestamp;
    405 }
    406 
    407 android_dataspace SurfaceTexture::getCurrentDataSpace() {
    408     SFT_LOGV("getCurrentDataSpace");
    409     Mutex::Autolock lock(mMutex);
    410     return mCurrentDataSpace;
    411 }
    412 
    413 uint64_t SurfaceTexture::getFrameNumber() {
    414     SFT_LOGV("getFrameNumber");
    415     Mutex::Autolock lock(mMutex);
    416     return mCurrentFrameNumber;
    417 }
    418 
    419 Rect SurfaceTexture::getCurrentCrop() const {
    420     Mutex::Autolock lock(mMutex);
    421     return (mCurrentScalingMode == NATIVE_WINDOW_SCALING_MODE_SCALE_CROP)
    422                    ? scaleDownCrop(mCurrentCrop, mDefaultWidth, mDefaultHeight)
    423                    : mCurrentCrop;
    424 }
    425 
    426 uint32_t SurfaceTexture::getCurrentTransform() const {
    427     Mutex::Autolock lock(mMutex);
    428     return mCurrentTransform;
    429 }
    430 
    431 uint32_t SurfaceTexture::getCurrentScalingMode() const {
    432     Mutex::Autolock lock(mMutex);
    433     return mCurrentScalingMode;
    434 }
    435 
    436 sp<Fence> SurfaceTexture::getCurrentFence() const {
    437     Mutex::Autolock lock(mMutex);
    438     return mCurrentFence;
    439 }
    440 
    441 std::shared_ptr<FenceTime> SurfaceTexture::getCurrentFenceTime() const {
    442     Mutex::Autolock lock(mMutex);
    443     return mCurrentFenceTime;
    444 }
    445 
    446 void SurfaceTexture::freeBufferLocked(int slotIndex) {
    447     SFT_LOGV("freeBufferLocked: slotIndex=%d", slotIndex);
    448     if (slotIndex == mCurrentTexture) {
    449         mCurrentTexture = BufferQueue::INVALID_BUFFER_SLOT;
    450     }
    451     // The slotIndex buffer could have EGL or SkImage cache, but there is no way to tell for sure.
    452     // Buffers can be freed after SurfaceTexture has detached from GL context or View.
    453     mImageConsumer.onFreeBufferLocked(slotIndex);
    454     mEGLConsumer.onFreeBufferLocked(slotIndex);
    455     ConsumerBase::freeBufferLocked(slotIndex);
    456 }
    457 
    458 void SurfaceTexture::abandonLocked() {
    459     SFT_LOGV("abandonLocked");
    460     mEGLConsumer.onAbandonLocked();
    461     ConsumerBase::abandonLocked();
    462 }
    463 
    464 status_t SurfaceTexture::setConsumerUsageBits(uint64_t usage) {
    465     return ConsumerBase::setConsumerUsageBits(usage | DEFAULT_USAGE_FLAGS);
    466 }
    467 
    468 void SurfaceTexture::dumpLocked(String8& result, const char* prefix) const {
    469     result.appendFormat(
    470             "%smTexName=%d mCurrentTexture=%d\n"
    471             "%smCurrentCrop=[%d,%d,%d,%d] mCurrentTransform=%#x\n",
    472             prefix, mTexName, mCurrentTexture, prefix, mCurrentCrop.left, mCurrentCrop.top,
    473             mCurrentCrop.right, mCurrentCrop.bottom, mCurrentTransform);
    474 
    475     ConsumerBase::dumpLocked(result, prefix);
    476 }
    477 
    478 sk_sp<SkImage> SurfaceTexture::dequeueImage(SkMatrix& transformMatrix, bool* queueEmpty,
    479                                             uirenderer::RenderState& renderState) {
    480     Mutex::Autolock _l(mMutex);
    481 
    482     if (mAbandoned) {
    483         SFT_LOGE("dequeueImage: SurfaceTexture is abandoned!");
    484         return nullptr;
    485     }
    486 
    487     if (mOpMode != OpMode::attachedToView) {
    488         SFT_LOGE("dequeueImage: SurfaceTexture is not attached to a View");
    489         return nullptr;
    490     }
    491 
    492     auto image = mImageConsumer.dequeueImage(queueEmpty, *this, renderState);
    493     if (image.get()) {
    494         uirenderer::mat4(mCurrentTransformMatrix).copyTo(transformMatrix);
    495     }
    496     return image;
    497 }
    498 
    499 }  // namespace android
    500