Home | History | Annotate | Download | only in client2 
      1 /*
      2  * Copyright (C) 2012 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #define LOG_TAG "Camera2-CallbackProcessor"
     18 #define ATRACE_TAG ATRACE_TAG_CAMERA
     19 //#define LOG_NDEBUG 0
     20 
     21 #include <utils/Log.h>
     22 #include <utils/Trace.h>
     23 #include <gui/Surface.h>
     24 
     25 #include "common/CameraDeviceBase.h"
     26 #include "api1/Camera2Client.h"
     27 #include "api1/client2/CallbackProcessor.h"
     28 
     29 #define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
     30 
     31 namespace android {
     32 namespace camera2 {
     33 
     34 CallbackProcessor::CallbackProcessor(sp<Camera2Client> client):
     35         Thread(false),
     36         mClient(client),
     37         mDevice(client->getCameraDevice()),
     38         mId(client->getCameraId()),
     39         mCallbackAvailable(false),
     40         mCallbackToApp(false),
     41         mCallbackStreamId(NO_STREAM) {
     42 }
     43 
     44 CallbackProcessor::~CallbackProcessor() {
     45     ALOGV("%s: Exit", __FUNCTION__);
     46     deleteStream();
     47 }
     48 
     49 void CallbackProcessor::onFrameAvailable(const BufferItem& /*item*/) {
     50     Mutex::Autolock l(mInputMutex);
     51     if (!mCallbackAvailable) {
     52         mCallbackAvailable = true;
     53         mCallbackAvailableSignal.signal();
     54     }
     55 }
     56 
     57 status_t CallbackProcessor::setCallbackWindow(
     58         const sp<Surface>& callbackWindow) {
     59     ATRACE_CALL();
     60     status_t res;
     61 
     62     Mutex::Autolock l(mInputMutex);
     63 
     64     sp<Camera2Client> client = mClient.promote();
     65     if (client == 0) return OK;
     66     sp<CameraDeviceBase> device = client->getCameraDevice();
     67 
     68     // If the window is changing, clear out stream if it already exists
     69     if (mCallbackWindow != callbackWindow && mCallbackStreamId != NO_STREAM) {
     70         res = device->deleteStream(mCallbackStreamId);
     71         if (res != OK) {
     72             ALOGE("%s: Camera %d: Unable to delete old stream "
     73                     "for callbacks: %s (%d)", __FUNCTION__,
     74                     client->getCameraId(), strerror(-res), res);
     75             return res;
     76         }
     77         mCallbackStreamId = NO_STREAM;
     78         mCallbackConsumer.clear();
     79     }
     80     mCallbackWindow = callbackWindow;
     81     mCallbackToApp = (mCallbackWindow != NULL);
     82 
     83     return OK;
     84 }
     85 
     86 status_t CallbackProcessor::updateStream(const Parameters &params) {
     87     ATRACE_CALL();
     88     status_t res;
     89 
     90     Mutex::Autolock l(mInputMutex);
     91 
     92     sp<CameraDeviceBase> device = mDevice.promote();
     93     if (device == 0) {
     94         ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
     95         return INVALID_OPERATION;
     96     }
     97 
     98     // If possible, use the flexible YUV format
     99     int32_t callbackFormat = params.previewFormat;
    100     if (mCallbackToApp) {
    101         // TODO: etalvala: This should use the flexible YUV format as well, but
    102         // need to reconcile HAL2/HAL3 requirements.
    103         callbackFormat = HAL_PIXEL_FORMAT_YV12;
    104     } else if(params.fastInfo.useFlexibleYuv &&
    105             (params.previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP ||
    106              params.previewFormat == HAL_PIXEL_FORMAT_YV12) ) {
    107         callbackFormat = HAL_PIXEL_FORMAT_YCbCr_420_888;
    108     }
    109 
    110     if (!mCallbackToApp && mCallbackConsumer == 0) {
    111         // Create CPU buffer queue endpoint, since app hasn't given us one
    112         // Make it async to avoid disconnect deadlocks
    113         sp<IGraphicBufferProducer> producer;
    114         sp<IGraphicBufferConsumer> consumer;
    115         BufferQueue::createBufferQueue(&producer, &consumer);
    116         mCallbackConsumer = new CpuConsumer(consumer, kCallbackHeapCount);
    117         mCallbackConsumer->setFrameAvailableListener(this);
    118         mCallbackConsumer->setName(String8("Camera2-CallbackConsumer"));
    119         mCallbackWindow = new Surface(producer);
    120     }
    121 
    122     if (mCallbackStreamId != NO_STREAM) {
    123         // Check if stream parameters have to change
    124         uint32_t currentWidth, currentHeight, currentFormat;
    125         res = device->getStreamInfo(mCallbackStreamId,
    126                 &currentWidth, &currentHeight, &currentFormat, 0);
    127         if (res != OK) {
    128             ALOGE("%s: Camera %d: Error querying callback output stream info: "
    129                     "%s (%d)", __FUNCTION__, mId,
    130                     strerror(-res), res);
    131             return res;
    132         }
    133         if (currentWidth != (uint32_t)params.previewWidth ||
    134                 currentHeight != (uint32_t)params.previewHeight ||
    135                 currentFormat != (uint32_t)callbackFormat) {
    136             // Since size should only change while preview is not running,
    137             // assuming that all existing use of old callback stream is
    138             // completed.
    139             ALOGV("%s: Camera %d: Deleting stream %d since the buffer "
    140                     "parameters changed", __FUNCTION__, mId, mCallbackStreamId);
    141             res = device->deleteStream(mCallbackStreamId);
    142             if (res != OK) {
    143                 ALOGE("%s: Camera %d: Unable to delete old output stream "
    144                         "for callbacks: %s (%d)", __FUNCTION__,
    145                         mId, strerror(-res), res);
    146                 return res;
    147             }
    148             mCallbackStreamId = NO_STREAM;
    149         }
    150     }
    151 
    152     if (mCallbackStreamId == NO_STREAM) {
    153         ALOGV("Creating callback stream: %d x %d, format 0x%x, API format 0x%x",
    154                 params.previewWidth, params.previewHeight,
    155                 callbackFormat, params.previewFormat);
    156         res = device->createStream(mCallbackWindow,
    157                 params.previewWidth, params.previewHeight, callbackFormat,
    158                 HAL_DATASPACE_V0_JFIF, CAMERA3_STREAM_ROTATION_0, &mCallbackStreamId);
    159         if (res != OK) {
    160             ALOGE("%s: Camera %d: Can't create output stream for callbacks: "
    161                     "%s (%d)", __FUNCTION__, mId,
    162                     strerror(-res), res);
    163             return res;
    164         }
    165     }
    166 
    167     return OK;
    168 }
    169 
    170 status_t CallbackProcessor::deleteStream() {
    171     ATRACE_CALL();
    172     sp<CameraDeviceBase> device;
    173     status_t res;
    174     {
    175         Mutex::Autolock l(mInputMutex);
    176 
    177         if (mCallbackStreamId == NO_STREAM) {
    178             return OK;
    179         }
    180         device = mDevice.promote();
    181         if (device == 0) {
    182             ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
    183             return INVALID_OPERATION;
    184         }
    185     }
    186     res = device->waitUntilDrained();
    187     if (res != OK) {
    188         ALOGE("%s: Error waiting for HAL to drain: %s (%d)",
    189                 __FUNCTION__, strerror(-res), res);
    190         return res;
    191     }
    192 
    193     res = device->deleteStream(mCallbackStreamId);
    194     if (res != OK) {
    195         ALOGE("%s: Unable to delete callback stream: %s (%d)",
    196                 __FUNCTION__, strerror(-res), res);
    197         return res;
    198     }
    199 
    200     {
    201         Mutex::Autolock l(mInputMutex);
    202 
    203         mCallbackHeap.clear();
    204         mCallbackWindow.clear();
    205         mCallbackConsumer.clear();
    206 
    207         mCallbackStreamId = NO_STREAM;
    208     }
    209     return OK;
    210 }
    211 
    212 int CallbackProcessor::getStreamId() const {
    213     Mutex::Autolock l(mInputMutex);
    214     return mCallbackStreamId;
    215 }
    216 
    217 void CallbackProcessor::dump(int /*fd*/, const Vector<String16>& /*args*/) const {
    218 }
    219 
    220 bool CallbackProcessor::threadLoop() {
    221     status_t res;
    222 
    223     {
    224         Mutex::Autolock l(mInputMutex);
    225         while (!mCallbackAvailable) {
    226             res = mCallbackAvailableSignal.waitRelative(mInputMutex,
    227                     kWaitDuration);
    228             if (res == TIMED_OUT) return true;
    229         }
    230         mCallbackAvailable = false;
    231     }
    232 
    233     do {
    234         sp<Camera2Client> client = mClient.promote();
    235         if (client == 0) {
    236             res = discardNewCallback();
    237         } else {
    238             res = processNewCallback(client);
    239         }
    240     } while (res == OK);
    241 
    242     return true;
    243 }
    244 
    245 status_t CallbackProcessor::discardNewCallback() {
    246     ATRACE_CALL();
    247     status_t res;
    248     CpuConsumer::LockedBuffer imgBuffer;
    249     res = mCallbackConsumer->lockNextBuffer(&imgBuffer);
    250     if (res != OK) {
    251         if (res != BAD_VALUE) {
    252             ALOGE("%s: Camera %d: Error receiving next callback buffer: "
    253                     "%s (%d)", __FUNCTION__, mId, strerror(-res), res);
    254         }
    255         return res;
    256     }
    257     mCallbackConsumer->unlockBuffer(imgBuffer);
    258     return OK;
    259 }
    260 
    261 status_t CallbackProcessor::processNewCallback(sp<Camera2Client> &client) {
    262     ATRACE_CALL();
    263     status_t res;
    264 
    265     sp<Camera2Heap> callbackHeap;
    266     bool useFlexibleYuv = false;
    267     int32_t previewFormat = 0;
    268     size_t heapIdx;
    269 
    270     {
    271         /* acquire SharedParameters before mMutex so we don't dead lock
    272             with Camera2Client code calling into StreamingProcessor */
    273         SharedParameters::Lock l(client->getParameters());
    274         Mutex::Autolock m(mInputMutex);
    275         CpuConsumer::LockedBuffer imgBuffer;
    276         if (mCallbackStreamId == NO_STREAM) {
    277             ALOGV("%s: Camera %d:No stream is available"
    278                     , __FUNCTION__, mId);
    279             return INVALID_OPERATION;
    280         }
    281 
    282         ALOGV("%s: Getting buffer", __FUNCTION__);
    283         res = mCallbackConsumer->lockNextBuffer(&imgBuffer);
    284         if (res != OK) {
    285             if (res != BAD_VALUE) {
    286                 ALOGE("%s: Camera %d: Error receiving next callback buffer: "
    287                         "%s (%d)", __FUNCTION__, mId, strerror(-res), res);
    288             }
    289             return res;
    290         }
    291         ALOGV("%s: Camera %d: Preview callback available", __FUNCTION__,
    292                 mId);
    293 
    294         if ( l.mParameters.state != Parameters::PREVIEW
    295                 && l.mParameters.state != Parameters::RECORD
    296                 && l.mParameters.state != Parameters::VIDEO_SNAPSHOT) {
    297             ALOGV("%s: Camera %d: No longer streaming",
    298                     __FUNCTION__, mId);
    299             mCallbackConsumer->unlockBuffer(imgBuffer);
    300             return OK;
    301         }
    302 
    303         if (! (l.mParameters.previewCallbackFlags &
    304                 CAMERA_FRAME_CALLBACK_FLAG_ENABLE_MASK) ) {
    305             ALOGV("%s: No longer enabled, dropping", __FUNCTION__);
    306             mCallbackConsumer->unlockBuffer(imgBuffer);
    307             return OK;
    308         }
    309         if ((l.mParameters.previewCallbackFlags &
    310                         CAMERA_FRAME_CALLBACK_FLAG_ONE_SHOT_MASK) &&
    311                 !l.mParameters.previewCallbackOneShot) {
    312             ALOGV("%s: One shot mode, already sent, dropping", __FUNCTION__);
    313             mCallbackConsumer->unlockBuffer(imgBuffer);
    314             return OK;
    315         }
    316 
    317         if (imgBuffer.width != static_cast<uint32_t>(l.mParameters.previewWidth) ||
    318                 imgBuffer.height != static_cast<uint32_t>(l.mParameters.previewHeight)) {
    319             ALOGW("%s: The preview size has changed to %d x %d from %d x %d, this buffer is"
    320                     " no longer valid, dropping",__FUNCTION__,
    321                     l.mParameters.previewWidth, l.mParameters.previewHeight,
    322                     imgBuffer.width, imgBuffer.height);
    323             mCallbackConsumer->unlockBuffer(imgBuffer);
    324             return OK;
    325         }
    326 
    327         previewFormat = l.mParameters.previewFormat;
    328         useFlexibleYuv = l.mParameters.fastInfo.useFlexibleYuv &&
    329                 (previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP ||
    330                  previewFormat == HAL_PIXEL_FORMAT_YV12);
    331 
    332         int32_t expectedFormat = useFlexibleYuv ?
    333                 HAL_PIXEL_FORMAT_YCbCr_420_888 : previewFormat;
    334 
    335         if (imgBuffer.format != expectedFormat) {
    336             ALOGE("%s: Camera %d: Unexpected format for callback: "
    337                     "0x%x, expected 0x%x", __FUNCTION__, mId,
    338                     imgBuffer.format, expectedFormat);
    339             mCallbackConsumer->unlockBuffer(imgBuffer);
    340             return INVALID_OPERATION;
    341         }
    342 
    343         // In one-shot mode, stop sending callbacks after the first one
    344         if (l.mParameters.previewCallbackFlags &
    345                 CAMERA_FRAME_CALLBACK_FLAG_ONE_SHOT_MASK) {
    346             ALOGV("%s: clearing oneshot", __FUNCTION__);
    347             l.mParameters.previewCallbackOneShot = false;
    348         }
    349 
    350         uint32_t destYStride = 0;
    351         uint32_t destCStride = 0;
    352         if (useFlexibleYuv) {
    353             if (previewFormat == HAL_PIXEL_FORMAT_YV12) {
    354                 // Strides must align to 16 for YV12
    355                 destYStride = ALIGN(imgBuffer.width, 16);
    356                 destCStride = ALIGN(destYStride / 2, 16);
    357             } else {
    358                 // No padding for NV21
    359                 ALOG_ASSERT(previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP,
    360                         "Unexpected preview format 0x%x", previewFormat);
    361                 destYStride = imgBuffer.width;
    362                 destCStride = destYStride / 2;
    363             }
    364         } else {
    365             destYStride = imgBuffer.stride;
    366             // don't care about cStride
    367         }
    368 
    369         size_t bufferSize = Camera2Client::calculateBufferSize(
    370                 imgBuffer.width, imgBuffer.height,
    371                 previewFormat, destYStride);
    372         size_t currentBufferSize = (mCallbackHeap == 0) ?
    373                 0 : (mCallbackHeap->mHeap->getSize() / kCallbackHeapCount);
    374         if (bufferSize != currentBufferSize) {
    375             mCallbackHeap.clear();
    376             mCallbackHeap = new Camera2Heap(bufferSize, kCallbackHeapCount,
    377                     "Camera2Client::CallbackHeap");
    378             if (mCallbackHeap->mHeap->getSize() == 0) {
    379                 ALOGE("%s: Camera %d: Unable to allocate memory for callbacks",
    380                         __FUNCTION__, mId);
    381                 mCallbackConsumer->unlockBuffer(imgBuffer);
    382                 return INVALID_OPERATION;
    383             }
    384 
    385             mCallbackHeapHead = 0;
    386             mCallbackHeapFree = kCallbackHeapCount;
    387         }
    388 
    389         if (mCallbackHeapFree == 0) {
    390             ALOGE("%s: Camera %d: No free callback buffers, dropping frame",
    391                     __FUNCTION__, mId);
    392             mCallbackConsumer->unlockBuffer(imgBuffer);
    393             return OK;
    394         }
    395 
    396         heapIdx = mCallbackHeapHead;
    397 
    398         mCallbackHeapHead = (mCallbackHeapHead + 1) % kCallbackHeapCount;
    399         mCallbackHeapFree--;
    400 
    401         // TODO: Get rid of this copy by passing the gralloc queue all the way
    402         // to app
    403 
    404         ssize_t offset;
    405         size_t size;
    406         sp<IMemoryHeap> heap =
    407                 mCallbackHeap->mBuffers[heapIdx]->getMemory(&offset,
    408                         &size);
    409         uint8_t *data = (uint8_t*)heap->getBase() + offset;
    410 
    411         if (!useFlexibleYuv) {
    412             // Can just memcpy when HAL format matches API format
    413             memcpy(data, imgBuffer.data, bufferSize);
    414         } else {
    415             res = convertFromFlexibleYuv(previewFormat, data, imgBuffer,
    416                     destYStride, destCStride);
    417             if (res != OK) {
    418                 ALOGE("%s: Camera %d: Can't convert between 0x%x and 0x%x formats!",
    419                         __FUNCTION__, mId, imgBuffer.format, previewFormat);
    420                 mCallbackConsumer->unlockBuffer(imgBuffer);
    421                 return BAD_VALUE;
    422             }
    423         }
    424 
    425         ALOGV("%s: Freeing buffer", __FUNCTION__);
    426         mCallbackConsumer->unlockBuffer(imgBuffer);
    427 
    428         // mCallbackHeap may get freed up once input mutex is released
    429         callbackHeap = mCallbackHeap;
    430     }
    431 
    432     // Call outside parameter lock to allow re-entrancy from notification
    433     {
    434         Camera2Client::SharedCameraCallbacks::Lock
    435             l(client->mSharedCameraCallbacks);
    436         if (l.mRemoteCallback != 0) {
    437             ALOGV("%s: Camera %d: Invoking client data callback",
    438                     __FUNCTION__, mId);
    439             l.mRemoteCallback->dataCallback(CAMERA_MSG_PREVIEW_FRAME,
    440                     callbackHeap->mBuffers[heapIdx], NULL);
    441         }
    442     }
    443 
    444     // Only increment free if we're still using the same heap
    445     mCallbackHeapFree++;
    446 
    447     ALOGV("%s: exit", __FUNCTION__);
    448 
    449     return OK;
    450 }
    451 
    452 status_t CallbackProcessor::convertFromFlexibleYuv(int32_t previewFormat,
    453         uint8_t *dst,
    454         const CpuConsumer::LockedBuffer &src,
    455         uint32_t dstYStride,
    456         uint32_t dstCStride) const {
    457 
    458     if (previewFormat != HAL_PIXEL_FORMAT_YCrCb_420_SP &&
    459             previewFormat != HAL_PIXEL_FORMAT_YV12) {
    460         ALOGE("%s: Camera %d: Unexpected preview format when using "
    461                 "flexible YUV: 0x%x", __FUNCTION__, mId, previewFormat);
    462         return INVALID_OPERATION;
    463     }
    464 
    465     // Copy Y plane, adjusting for stride
    466     const uint8_t *ySrc = src.data;
    467     uint8_t *yDst = dst;
    468     for (size_t row = 0; row < src.height; row++) {
    469         memcpy(yDst, ySrc, src.width);
    470         ySrc += src.stride;
    471         yDst += dstYStride;
    472     }
    473 
    474     // Copy/swizzle chroma planes, 4:2:0 subsampling
    475     const uint8_t *cbSrc = src.dataCb;
    476     const uint8_t *crSrc = src.dataCr;
    477     size_t chromaHeight = src.height / 2;
    478     size_t chromaWidth = src.width / 2;
    479     ssize_t chromaGap = src.chromaStride -
    480             (chromaWidth * src.chromaStep);
    481     size_t dstChromaGap = dstCStride - chromaWidth;
    482 
    483     if (previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) {
    484         // Flexible YUV chroma to NV21 chroma
    485         uint8_t *crcbDst = yDst;
    486         // Check for shortcuts
    487         if (cbSrc == crSrc + 1 && src.chromaStep == 2) {
    488             ALOGV("%s: Fast NV21->NV21", __FUNCTION__);
    489             // Source has semiplanar CrCb chroma layout, can copy by rows
    490             for (size_t row = 0; row < chromaHeight; row++) {
    491                 memcpy(crcbDst, crSrc, src.width);
    492                 crcbDst += src.width;
    493                 crSrc += src.chromaStride;
    494             }
    495         } else {
    496             ALOGV("%s: Generic->NV21", __FUNCTION__);
    497             // Generic copy, always works but not very efficient
    498             for (size_t row = 0; row < chromaHeight; row++) {
    499                 for (size_t col = 0; col < chromaWidth; col++) {
    500                     *(crcbDst++) = *crSrc;
    501                     *(crcbDst++) = *cbSrc;
    502                     crSrc += src.chromaStep;
    503                     cbSrc += src.chromaStep;
    504                 }
    505                 crSrc += chromaGap;
    506                 cbSrc += chromaGap;
    507             }
    508         }
    509     } else {
    510         // flexible YUV chroma to YV12 chroma
    511         ALOG_ASSERT(previewFormat == HAL_PIXEL_FORMAT_YV12,
    512                 "Unexpected preview format 0x%x", previewFormat);
    513         uint8_t *crDst = yDst;
    514         uint8_t *cbDst = yDst + chromaHeight * dstCStride;
    515         if (src.chromaStep == 1) {
    516             ALOGV("%s: Fast YV12->YV12", __FUNCTION__);
    517             // Source has planar chroma layout, can copy by row
    518             for (size_t row = 0; row < chromaHeight; row++) {
    519                 memcpy(crDst, crSrc, chromaWidth);
    520                 crDst += dstCStride;
    521                 crSrc += src.chromaStride;
    522             }
    523             for (size_t row = 0; row < chromaHeight; row++) {
    524                 memcpy(cbDst, cbSrc, chromaWidth);
    525                 cbDst += dstCStride;
    526                 cbSrc += src.chromaStride;
    527             }
    528         } else {
    529             ALOGV("%s: Generic->YV12", __FUNCTION__);
    530             // Generic copy, always works but not very efficient
    531             for (size_t row = 0; row < chromaHeight; row++) {
    532                 for (size_t col = 0; col < chromaWidth; col++) {
    533                     *(crDst++) = *crSrc;
    534                     *(cbDst++) = *cbSrc;
    535                     crSrc += src.chromaStep;
    536                     cbSrc += src.chromaStep;
    537                 }
    538                 crSrc += chromaGap;
    539                 cbSrc += chromaGap;
    540                 crDst += dstChromaGap;
    541                 cbDst += dstChromaGap;
    542             }
    543         }
    544     }
    545 
    546     return OK;
    547 }
    548 
    549 }; // namespace camera2
    550 }; // namespace android
    551