Home | History | Annotate | Download | only in ndk 
      1 /*
      2  * Copyright (C) 2016 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #include <inttypes.h>
     18 
     19 //#define LOG_NDEBUG 0
     20 #define LOG_TAG "NdkImageReader"
     21 
     22 #include "NdkImagePriv.h"
     23 #include "NdkImageReaderPriv.h"
     24 
     25 #include <utils/Log.h>
     26 #include <android_runtime/android_view_Surface.h>
     27 
     28 using namespace android;
     29 
     30 namespace {
     31     // Get an ID that's unique within this process.
     32     static int32_t createProcessUniqueId() {
     33         static volatile int32_t globalCounter = 0;
     34         return android_atomic_inc(&globalCounter);
     35     }
     36 }
     37 
     38 const char* AImageReader::kCallbackFpKey = "Callback";
     39 const char* AImageReader::kContextKey    = "Context";
     40 
     41 bool
     42 AImageReader::isSupportedFormat(int32_t format) {
     43     switch (format) {
     44         case AIMAGE_FORMAT_YUV_420_888:
     45         case AIMAGE_FORMAT_JPEG:
     46         case AIMAGE_FORMAT_RAW16:
     47         case AIMAGE_FORMAT_RAW_PRIVATE:
     48         case AIMAGE_FORMAT_RAW10:
     49         case AIMAGE_FORMAT_RAW12:
     50         case AIMAGE_FORMAT_DEPTH16:
     51         case AIMAGE_FORMAT_DEPTH_POINT_CLOUD:
     52             return true;
     53         default:
     54             return false;
     55     }
     56 }
     57 
     58 int
     59 AImageReader::getNumPlanesForFormat(int32_t format) {
     60     switch (format) {
     61         case AIMAGE_FORMAT_YUV_420_888:
     62             return 3;
     63         case AIMAGE_FORMAT_JPEG:
     64         case AIMAGE_FORMAT_RAW16:
     65         case AIMAGE_FORMAT_RAW_PRIVATE:
     66         case AIMAGE_FORMAT_RAW10:
     67         case AIMAGE_FORMAT_RAW12:
     68         case AIMAGE_FORMAT_DEPTH16:
     69         case AIMAGE_FORMAT_DEPTH_POINT_CLOUD:
     70             return 1;
     71         default:
     72             return -1;
     73     }
     74 }
     75 
     76 void
     77 AImageReader::FrameListener::onFrameAvailable(const BufferItem& /*item*/) {
     78     Mutex::Autolock _l(mLock);
     79     sp<AImageReader> reader = mReader.promote();
     80     if (reader == nullptr) {
     81         ALOGW("A frame is available after AImageReader closed!");
     82         return; // reader has been closed
     83     }
     84     if (mListener.onImageAvailable == nullptr) {
     85         return; // No callback registered
     86     }
     87 
     88     sp<AMessage> msg = new AMessage(AImageReader::kWhatImageAvailable, reader->mHandler);
     89     msg->setPointer(AImageReader::kCallbackFpKey, (void *) mListener.onImageAvailable);
     90     msg->setPointer(AImageReader::kContextKey, mListener.context);
     91     msg->post();
     92 }
     93 
     94 media_status_t
     95 AImageReader::FrameListener::setImageListener(AImageReader_ImageListener* listener) {
     96     Mutex::Autolock _l(mLock);
     97     if (listener == nullptr) {
     98         mListener.context = nullptr;
     99         mListener.onImageAvailable = nullptr;
    100     } else {
    101         mListener = *listener;
    102     }
    103     return AMEDIA_OK;
    104 }
    105 
    106 media_status_t
    107 AImageReader::setImageListenerLocked(AImageReader_ImageListener* listener) {
    108     return mFrameListener->setImageListener(listener);
    109 }
    110 
    111 media_status_t
    112 AImageReader::setImageListener(AImageReader_ImageListener* listener) {
    113     Mutex::Autolock _l(mLock);
    114     return setImageListenerLocked(listener);
    115 }
    116 
    117 void AImageReader::CallbackHandler::onMessageReceived(
    118         const sp<AMessage> &msg) {
    119     switch (msg->what()) {
    120         case kWhatImageAvailable:
    121         {
    122             AImageReader_ImageCallback onImageAvailable;
    123             void* context;
    124             bool found = msg->findPointer(kCallbackFpKey, (void**) &onImageAvailable);
    125             if (!found || onImageAvailable == nullptr) {
    126                 ALOGE("%s: Cannot find onImageAvailable callback fp!", __FUNCTION__);
    127                 return;
    128             }
    129             found = msg->findPointer(kContextKey, &context);
    130             if (!found) {
    131                 ALOGE("%s: Cannot find callback context!", __FUNCTION__);
    132                 return;
    133             }
    134             (*onImageAvailable)(context, mReader);
    135             break;
    136         }
    137         default:
    138             ALOGE("%s: unknown message type %d", __FUNCTION__, msg->what());
    139             break;
    140     }
    141 }
    142 
    143 AImageReader::AImageReader(int32_t width, int32_t height, int32_t format, int32_t maxImages) :
    144         mWidth(width), mHeight(height), mFormat(format), mMaxImages(maxImages),
    145         mNumPlanes(getNumPlanesForFormat(format)),
    146         mFrameListener(new FrameListener(this)) {}
    147 
    148 media_status_t
    149 AImageReader::init() {
    150     PublicFormat publicFormat = static_cast<PublicFormat>(mFormat);
    151     mHalFormat = android_view_Surface_mapPublicFormatToHalFormat(publicFormat);
    152     mHalDataSpace = android_view_Surface_mapPublicFormatToHalDataspace(publicFormat);
    153 
    154     sp<IGraphicBufferProducer> gbProducer;
    155     sp<IGraphicBufferConsumer> gbConsumer;
    156     BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
    157 
    158     sp<CpuConsumer> cpuConsumer;
    159     String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d-%d",
    160             mWidth, mHeight, mFormat, mMaxImages, getpid(),
    161             createProcessUniqueId());
    162 
    163     cpuConsumer = new CpuConsumer(gbConsumer, mMaxImages, /*controlledByApp*/true);
    164     if (cpuConsumer == nullptr) {
    165         ALOGE("Failed to allocate CpuConsumer");
    166         return AMEDIA_ERROR_UNKNOWN;
    167     }
    168 
    169     mCpuConsumer = cpuConsumer;
    170     mCpuConsumer->setName(consumerName);
    171     mProducer = gbProducer;
    172 
    173     sp<ConsumerBase> consumer = cpuConsumer;
    174     consumer->setFrameAvailableListener(mFrameListener);
    175 
    176     status_t res;
    177     res = cpuConsumer->setDefaultBufferSize(mWidth, mHeight);
    178     if (res != OK) {
    179         ALOGE("Failed to set CpuConsumer buffer size");
    180         return AMEDIA_ERROR_UNKNOWN;
    181     }
    182     res = cpuConsumer->setDefaultBufferFormat(mHalFormat);
    183     if (res != OK) {
    184         ALOGE("Failed to set CpuConsumer buffer format");
    185         return AMEDIA_ERROR_UNKNOWN;
    186     }
    187     res = cpuConsumer->setDefaultBufferDataSpace(mHalDataSpace);
    188     if (res != OK) {
    189         ALOGE("Failed to set CpuConsumer buffer dataSpace");
    190         return AMEDIA_ERROR_UNKNOWN;
    191     }
    192 
    193     mSurface = new Surface(mProducer, /*controlledByApp*/true);
    194     if (mSurface == nullptr) {
    195         ALOGE("Failed to create surface");
    196         return AMEDIA_ERROR_UNKNOWN;
    197     }
    198     mWindow = static_cast<ANativeWindow*>(mSurface.get());
    199 
    200     for (int i = 0; i < mMaxImages; i++) {
    201         CpuConsumer::LockedBuffer* buffer = new CpuConsumer::LockedBuffer;
    202         mBuffers.push_back(buffer);
    203     }
    204 
    205     mCbLooper = new ALooper;
    206     mCbLooper->setName(consumerName.string());
    207     res = mCbLooper->start(
    208             /*runOnCallingThread*/false,
    209             /*canCallJava*/       true,
    210             PRIORITY_DEFAULT);
    211     if (res != OK) {
    212         ALOGE("Failed to start the looper");
    213         return AMEDIA_ERROR_UNKNOWN;
    214     }
    215     mHandler = new CallbackHandler(this);
    216     mCbLooper->registerHandler(mHandler);
    217 
    218     return AMEDIA_OK;
    219 }
    220 
    221 AImageReader::~AImageReader() {
    222     Mutex::Autolock _l(mLock);
    223     AImageReader_ImageListener nullListener = {nullptr, nullptr};
    224     setImageListenerLocked(&nullListener);
    225 
    226     if (mCbLooper != nullptr) {
    227         mCbLooper->unregisterHandler(mHandler->id());
    228         mCbLooper->stop();
    229     }
    230     mCbLooper.clear();
    231     mHandler.clear();
    232 
    233     // Close all previously acquired images
    234     for (auto it = mAcquiredImages.begin();
    235               it != mAcquiredImages.end(); it++) {
    236         AImage* image = *it;
    237         image->close();
    238     }
    239 
    240     // Delete LockedBuffers
    241     for (auto it = mBuffers.begin();
    242               it != mBuffers.end(); it++) {
    243         delete *it;
    244     }
    245 
    246     if (mCpuConsumer != nullptr) {
    247         mCpuConsumer->abandon();
    248         mCpuConsumer->setFrameAvailableListener(nullptr);
    249     }
    250 }
    251 
    252 media_status_t
    253 AImageReader::acquireCpuConsumerImageLocked(/*out*/AImage** image) {
    254     *image = nullptr;
    255     CpuConsumer::LockedBuffer* buffer = getLockedBufferLocked();
    256     if (buffer == nullptr) {
    257         ALOGW("Unable to acquire a lockedBuffer, very likely client tries to lock more than"
    258             " maxImages buffers");
    259         return AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED;
    260     }
    261 
    262     status_t res = mCpuConsumer->lockNextBuffer(buffer);
    263     if (res != NO_ERROR) {
    264         returnLockedBufferLocked(buffer);
    265         if (res != BAD_VALUE /*no buffers*/) {
    266             if (res == NOT_ENOUGH_DATA) {
    267                 return AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED;
    268             } else {
    269                 ALOGE("%s Fail to lockNextBuffer with error: %d ",
    270                       __FUNCTION__, res);
    271                 return AMEDIA_ERROR_UNKNOWN;
    272             }
    273         }
    274         return AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE;
    275     }
    276 
    277     if (buffer->flexFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) {
    278         ALOGE("NV21 format is not supported by AImageReader");
    279         return AMEDIA_ERROR_UNSUPPORTED;
    280     }
    281 
    282     // Check if the left-top corner of the crop rect is origin, we currently assume this point is
    283     // zero, will revist this once this assumption turns out problematic.
    284     Point lt = buffer->crop.leftTop();
    285     if (lt.x != 0 || lt.y != 0) {
    286         ALOGE("crop left top corner [%d, %d] need to be at origin", lt.x, lt.y);
    287         return AMEDIA_ERROR_UNKNOWN;
    288     }
    289 
    290     // Check if the producer buffer configurations match what ImageReader configured.
    291     int outputWidth = getBufferWidth(buffer);
    292     int outputHeight = getBufferHeight(buffer);
    293 
    294     int readerFmt = mHalFormat;
    295     int readerWidth = mWidth;
    296     int readerHeight = mHeight;
    297 
    298     if ((buffer->format != HAL_PIXEL_FORMAT_BLOB) && (readerFmt != HAL_PIXEL_FORMAT_BLOB) &&
    299             (readerWidth != outputWidth || readerHeight != outputHeight)) {
    300         ALOGW("%s: Producer buffer size: %dx%d, doesn't match AImageReader configured size: %dx%d",
    301                 __FUNCTION__, outputWidth, outputHeight, readerWidth, readerHeight);
    302     }
    303 
    304     int bufFmt = buffer->format;
    305     if (readerFmt == HAL_PIXEL_FORMAT_YCbCr_420_888) {
    306         bufFmt = buffer->flexFormat;
    307     }
    308 
    309     if (readerFmt != bufFmt) {
    310         if (readerFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 && (bufFmt ==
    311                 HAL_PIXEL_FORMAT_YCrCb_420_SP || bufFmt == HAL_PIXEL_FORMAT_YV12)) {
    312             // Special casing for when producer switches to a format compatible with flexible YUV
    313             // (HAL_PIXEL_FORMAT_YCbCr_420_888).
    314             mHalFormat = bufFmt;
    315             ALOGD("%s: Overriding buffer format YUV_420_888 to %x.", __FUNCTION__, bufFmt);
    316         } else {
    317             // Return the buffer to the queue.
    318             mCpuConsumer->unlockBuffer(*buffer);
    319             returnLockedBufferLocked(buffer);
    320 
    321             ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
    322                     buffer->format, readerFmt);
    323 
    324             return AMEDIA_ERROR_UNKNOWN;
    325         }
    326     }
    327 
    328     if (mHalFormat == HAL_PIXEL_FORMAT_BLOB) {
    329         *image = new AImage(this, mFormat, buffer, buffer->timestamp,
    330                             readerWidth, readerHeight, mNumPlanes);
    331     } else {
    332         *image = new AImage(this, mFormat, buffer, buffer->timestamp,
    333                             outputWidth, outputHeight, mNumPlanes);
    334     }
    335     mAcquiredImages.push_back(*image);
    336     return AMEDIA_OK;
    337 }
    338 
    339 CpuConsumer::LockedBuffer*
    340 AImageReader::getLockedBufferLocked() {
    341     if (mBuffers.empty()) {
    342         return nullptr;
    343     }
    344     // Return a LockedBuffer pointer and remove it from the list
    345     auto it = mBuffers.begin();
    346     CpuConsumer::LockedBuffer* buffer = *it;
    347     mBuffers.erase(it);
    348     return buffer;
    349 }
    350 
    351 void
    352 AImageReader::returnLockedBufferLocked(CpuConsumer::LockedBuffer* buffer) {
    353     mBuffers.push_back(buffer);
    354 }
    355 
    356 void
    357 AImageReader::releaseImageLocked(AImage* image) {
    358     CpuConsumer::LockedBuffer* buffer = image->mBuffer;
    359     if (buffer == nullptr) {
    360         // This should not happen, but is not fatal
    361         ALOGW("AImage %p has no buffer!", image);
    362         return;
    363     }
    364 
    365     mCpuConsumer->unlockBuffer(*buffer);
    366     returnLockedBufferLocked(buffer);
    367     image->mBuffer = nullptr;
    368 
    369     bool found = false;
    370     // cleanup acquired image list
    371     for (auto it = mAcquiredImages.begin();
    372               it != mAcquiredImages.end(); it++) {
    373         AImage* readerCopy = *it;
    374         if (readerCopy == image) {
    375             found = true;
    376             mAcquiredImages.erase(it);
    377             break;
    378         }
    379     }
    380     if (!found) {
    381         ALOGE("Error: AImage %p is not generated by AImageReader %p",
    382                 image, this);
    383     }
    384 }
    385 
    386 int
    387 AImageReader::getBufferWidth(CpuConsumer::LockedBuffer* buffer) {
    388     if (buffer == nullptr) return -1;
    389 
    390     if (!buffer->crop.isEmpty()) {
    391         return buffer->crop.getWidth();
    392     }
    393     return buffer->width;
    394 }
    395 
    396 int
    397 AImageReader::getBufferHeight(CpuConsumer::LockedBuffer* buffer) {
    398     if (buffer == nullptr) return -1;
    399 
    400     if (!buffer->crop.isEmpty()) {
    401         return buffer->crop.getHeight();
    402     }
    403     return buffer->height;
    404 }
    405 
    406 media_status_t
    407 AImageReader::acquireNextImage(/*out*/AImage** image) {
    408     Mutex::Autolock _l(mLock);
    409     return acquireCpuConsumerImageLocked(image);
    410 }
    411 
    412 media_status_t
    413 AImageReader::acquireLatestImage(/*out*/AImage** image) {
    414     if (image == nullptr) {
    415         return AMEDIA_ERROR_INVALID_PARAMETER;
    416     }
    417     Mutex::Autolock _l(mLock);
    418     *image = nullptr;
    419     AImage* prevImage = nullptr;
    420     AImage* nextImage = nullptr;
    421     media_status_t ret = acquireCpuConsumerImageLocked(&prevImage);
    422     if (prevImage == nullptr) {
    423         return ret;
    424     }
    425     for (;;) {
    426         ret = acquireCpuConsumerImageLocked(&nextImage);
    427         if (nextImage == nullptr) {
    428             *image = prevImage;
    429             return AMEDIA_OK;
    430         }
    431         prevImage->close();
    432         prevImage->free();
    433         prevImage = nextImage;
    434         nextImage = nullptr;
    435     }
    436 }
    437 
    438 EXPORT
    439 media_status_t AImageReader_new(
    440         int32_t width, int32_t height, int32_t format, int32_t maxImages,
    441         /*out*/AImageReader** reader) {
    442     ALOGV("%s", __FUNCTION__);
    443 
    444     if (width < 1 || height < 1) {
    445         ALOGE("%s: image dimension must be positive: w:%d h:%d",
    446                 __FUNCTION__, width, height);
    447         return AMEDIA_ERROR_INVALID_PARAMETER;
    448     }
    449 
    450     if (maxImages < 1) {
    451         ALOGE("%s: max outstanding image count must be at least 1 (%d)",
    452                 __FUNCTION__, maxImages);
    453         return AMEDIA_ERROR_INVALID_PARAMETER;
    454     }
    455 
    456     if (!AImageReader::isSupportedFormat(format)) {
    457         ALOGE("%s: format %d is not supported by AImageReader",
    458                 __FUNCTION__, format);
    459         return AMEDIA_ERROR_INVALID_PARAMETER;
    460     }
    461 
    462     if (reader == nullptr) {
    463         ALOGE("%s: reader argument is null", __FUNCTION__);
    464         return AMEDIA_ERROR_INVALID_PARAMETER;
    465     }
    466 
    467     //*reader = new AImageReader(width, height, format, maxImages);
    468     AImageReader* tmpReader = new AImageReader(width, height, format, maxImages);
    469     if (tmpReader == nullptr) {
    470         ALOGE("%s: AImageReader allocation failed", __FUNCTION__);
    471         return AMEDIA_ERROR_UNKNOWN;
    472     }
    473     media_status_t ret = tmpReader->init();
    474     if (ret != AMEDIA_OK) {
    475         ALOGE("%s: AImageReader initialization failed!", __FUNCTION__);
    476         delete tmpReader;
    477         return ret;
    478     }
    479     *reader = tmpReader;
    480     (*reader)->incStrong((void*) AImageReader_new);
    481     return AMEDIA_OK;
    482 }
    483 
    484 EXPORT
    485 void AImageReader_delete(AImageReader* reader) {
    486     ALOGV("%s", __FUNCTION__);
    487     if (reader != nullptr) {
    488         reader->decStrong((void*) AImageReader_delete);
    489     }
    490     return;
    491 }
    492 
    493 EXPORT
    494 media_status_t AImageReader_getWindow(AImageReader* reader, /*out*/ANativeWindow** window) {
    495     ALOGE("%s", __FUNCTION__);
    496     if (reader == nullptr || window == nullptr) {
    497         ALOGE("%s: invalid argument. reader %p, window %p",
    498                 __FUNCTION__, reader, window);
    499         return AMEDIA_ERROR_INVALID_PARAMETER;
    500     }
    501     *window = reader->getWindow();
    502     return AMEDIA_OK;
    503 }
    504 
    505 EXPORT
    506 media_status_t AImageReader_getWidth(const AImageReader* reader, /*out*/int32_t* width) {
    507     ALOGV("%s", __FUNCTION__);
    508     if (reader == nullptr || width == nullptr) {
    509         ALOGE("%s: invalid argument. reader %p, width %p",
    510                 __FUNCTION__, reader, width);
    511         return AMEDIA_ERROR_INVALID_PARAMETER;
    512     }
    513     *width = reader->getWidth();
    514     return AMEDIA_OK;
    515 }
    516 
    517 EXPORT
    518 media_status_t AImageReader_getHeight(const AImageReader* reader, /*out*/int32_t* height) {
    519     ALOGV("%s", __FUNCTION__);
    520     if (reader == nullptr || height == nullptr) {
    521         ALOGE("%s: invalid argument. reader %p, height %p",
    522                 __FUNCTION__, reader, height);
    523         return AMEDIA_ERROR_INVALID_PARAMETER;
    524     }
    525     *height = reader->getHeight();
    526     return AMEDIA_OK;
    527 }
    528 
    529 EXPORT
    530 media_status_t AImageReader_getFormat(const AImageReader* reader, /*out*/int32_t* format) {
    531     ALOGV("%s", __FUNCTION__);
    532     if (reader == nullptr || format == nullptr) {
    533         ALOGE("%s: invalid argument. reader %p, format %p",
    534                 __FUNCTION__, reader, format);
    535         return AMEDIA_ERROR_INVALID_PARAMETER;
    536     }
    537     *format = reader->getFormat();
    538     return AMEDIA_OK;
    539 }
    540 
    541 EXPORT
    542 media_status_t AImageReader_getMaxImages(const AImageReader* reader, /*out*/int32_t* maxImages) {
    543     ALOGV("%s", __FUNCTION__);
    544     if (reader == nullptr || maxImages == nullptr) {
    545         ALOGE("%s: invalid argument. reader %p, maxImages %p",
    546                 __FUNCTION__, reader, maxImages);
    547         return AMEDIA_ERROR_INVALID_PARAMETER;
    548     }
    549     *maxImages = reader->getMaxImages();
    550     return AMEDIA_OK;
    551 }
    552 
    553 EXPORT
    554 media_status_t AImageReader_acquireNextImage(AImageReader* reader, /*out*/AImage** image) {
    555     ALOGV("%s", __FUNCTION__);
    556     if (reader == nullptr || image == nullptr) {
    557         ALOGE("%s: invalid argument. reader %p, maxImages %p",
    558                 __FUNCTION__, reader, image);
    559         return AMEDIA_ERROR_INVALID_PARAMETER;
    560     }
    561     return reader->acquireNextImage(image);
    562 }
    563 
    564 EXPORT
    565 media_status_t AImageReader_acquireLatestImage(AImageReader* reader, /*out*/AImage** image) {
    566     ALOGV("%s", __FUNCTION__);
    567     if (reader == nullptr || image == nullptr) {
    568         ALOGE("%s: invalid argument. reader %p, maxImages %p",
    569                 __FUNCTION__, reader, image);
    570         return AMEDIA_ERROR_INVALID_PARAMETER;
    571     }
    572     return reader->acquireLatestImage(image);
    573 }
    574 
    575 EXPORT
    576 media_status_t AImageReader_setImageListener(
    577         AImageReader* reader, AImageReader_ImageListener* listener) {
    578     ALOGV("%s", __FUNCTION__);
    579     if (reader == nullptr) {
    580         ALOGE("%s: invalid argument! reader %p", __FUNCTION__, reader);
    581         return AMEDIA_ERROR_INVALID_PARAMETER;
    582     }
    583 
    584     reader->setImageListener(listener);
    585     return AMEDIA_OK;
    586 }
    587