Home | History | Annotate | Download | only in client2 
      1 /*
      2  * Copyright (C) 2012 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #define LOG_TAG "Camera2-CallbackProcessor"
     18 #define ATRACE_TAG ATRACE_TAG_CAMERA
     19 //#define LOG_NDEBUG 0
     20 
     21 #include <utils/Log.h>
     22 #include <utils/Trace.h>
     23 #include <gui/Surface.h>
     24 
     25 #include "common/CameraDeviceBase.h"
     26 #include "api1/Camera2Client.h"
     27 #include "api1/client2/CallbackProcessor.h"
     28 
     29 #define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
     30 
     31 namespace android {
     32 namespace camera2 {
     33 
     34 CallbackProcessor::CallbackProcessor(sp<Camera2Client> client):
     35         Thread(false),
     36         mClient(client),
     37         mDevice(client->getCameraDevice()),
     38         mId(client->getCameraId()),
     39         mCallbackAvailable(false),
     40         mCallbackToApp(false),
     41         mCallbackStreamId(NO_STREAM) {
     42 }
     43 
     44 CallbackProcessor::~CallbackProcessor() {
     45     ALOGV("%s: Exit", __FUNCTION__);
     46     deleteStream();
     47 }
     48 
     49 void CallbackProcessor::onFrameAvailable() {
     50     Mutex::Autolock l(mInputMutex);
     51     if (!mCallbackAvailable) {
     52         mCallbackAvailable = true;
     53         mCallbackAvailableSignal.signal();
     54     }
     55 }
     56 
     57 status_t CallbackProcessor::setCallbackWindow(
     58         sp<ANativeWindow> callbackWindow) {
     59     ATRACE_CALL();
     60     status_t res;
     61 
     62     Mutex::Autolock l(mInputMutex);
     63 
     64     sp<Camera2Client> client = mClient.promote();
     65     if (client == 0) return OK;
     66     sp<CameraDeviceBase> device = client->getCameraDevice();
     67 
     68     // If the window is changing, clear out stream if it already exists
     69     if (mCallbackWindow != callbackWindow && mCallbackStreamId != NO_STREAM) {
     70         res = device->deleteStream(mCallbackStreamId);
     71         if (res != OK) {
     72             ALOGE("%s: Camera %d: Unable to delete old stream "
     73                     "for callbacks: %s (%d)", __FUNCTION__,
     74                     client->getCameraId(), strerror(-res), res);
     75             return res;
     76         }
     77         mCallbackStreamId = NO_STREAM;
     78         mCallbackConsumer.clear();
     79     }
     80     mCallbackWindow = callbackWindow;
     81     mCallbackToApp = (mCallbackWindow != NULL);
     82 
     83     return OK;
     84 }
     85 
     86 status_t CallbackProcessor::updateStream(const Parameters &params) {
     87     ATRACE_CALL();
     88     status_t res;
     89 
     90     Mutex::Autolock l(mInputMutex);
     91 
     92     sp<CameraDeviceBase> device = mDevice.promote();
     93     if (device == 0) {
     94         ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
     95         return INVALID_OPERATION;
     96     }
     97 
     98     // If possible, use the flexible YUV format
     99     int32_t callbackFormat = params.previewFormat;
    100     if (mCallbackToApp) {
    101         // TODO: etalvala: This should use the flexible YUV format as well, but
    102         // need to reconcile HAL2/HAL3 requirements.
    103         callbackFormat = HAL_PIXEL_FORMAT_YV12;
    104     } else if(params.fastInfo.useFlexibleYuv &&
    105             (params.previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP ||
    106              params.previewFormat == HAL_PIXEL_FORMAT_YV12) ) {
    107         callbackFormat = HAL_PIXEL_FORMAT_YCbCr_420_888;
    108     }
    109 
    110     if (!mCallbackToApp && mCallbackConsumer == 0) {
    111         // Create CPU buffer queue endpoint, since app hasn't given us one
    112         // Make it async to avoid disconnect deadlocks
    113         sp<BufferQueue> bq = new BufferQueue();
    114         mCallbackConsumer = new CpuConsumer(bq, kCallbackHeapCount);
    115         mCallbackConsumer->setFrameAvailableListener(this);
    116         mCallbackConsumer->setName(String8("Camera2Client::CallbackConsumer"));
    117         mCallbackWindow = new Surface(bq);
    118     }
    119 
    120     if (mCallbackStreamId != NO_STREAM) {
    121         // Check if stream parameters have to change
    122         uint32_t currentWidth, currentHeight, currentFormat;
    123         res = device->getStreamInfo(mCallbackStreamId,
    124                 &currentWidth, &currentHeight, &currentFormat);
    125         if (res != OK) {
    126             ALOGE("%s: Camera %d: Error querying callback output stream info: "
    127                     "%s (%d)", __FUNCTION__, mId,
    128                     strerror(-res), res);
    129             return res;
    130         }
    131         if (currentWidth != (uint32_t)params.previewWidth ||
    132                 currentHeight != (uint32_t)params.previewHeight ||
    133                 currentFormat != (uint32_t)callbackFormat) {
    134             // Since size should only change while preview is not running,
    135             // assuming that all existing use of old callback stream is
    136             // completed.
    137             ALOGV("%s: Camera %d: Deleting stream %d since the buffer "
    138                     "parameters changed", __FUNCTION__, mId, mCallbackStreamId);
    139             res = device->deleteStream(mCallbackStreamId);
    140             if (res != OK) {
    141                 ALOGE("%s: Camera %d: Unable to delete old output stream "
    142                         "for callbacks: %s (%d)", __FUNCTION__,
    143                         mId, strerror(-res), res);
    144                 return res;
    145             }
    146             mCallbackStreamId = NO_STREAM;
    147         }
    148     }
    149 
    150     if (mCallbackStreamId == NO_STREAM) {
    151         ALOGV("Creating callback stream: %d x %d, format 0x%x, API format 0x%x",
    152                 params.previewWidth, params.previewHeight,
    153                 callbackFormat, params.previewFormat);
    154         res = device->createStream(mCallbackWindow,
    155                 params.previewWidth, params.previewHeight,
    156                 callbackFormat, 0, &mCallbackStreamId);
    157         if (res != OK) {
    158             ALOGE("%s: Camera %d: Can't create output stream for callbacks: "
    159                     "%s (%d)", __FUNCTION__, mId,
    160                     strerror(-res), res);
    161             return res;
    162         }
    163     }
    164 
    165     return OK;
    166 }
    167 
    168 status_t CallbackProcessor::deleteStream() {
    169     ATRACE_CALL();
    170     sp<CameraDeviceBase> device;
    171     status_t res;
    172     {
    173         Mutex::Autolock l(mInputMutex);
    174 
    175         if (mCallbackStreamId == NO_STREAM) {
    176             return OK;
    177         }
    178         device = mDevice.promote();
    179         if (device == 0) {
    180             ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
    181             return INVALID_OPERATION;
    182         }
    183     }
    184     res = device->waitUntilDrained();
    185     if (res != OK) {
    186         ALOGE("%s: Error waiting for HAL to drain: %s (%d)",
    187                 __FUNCTION__, strerror(-res), res);
    188         return res;
    189     }
    190 
    191     res = device->deleteStream(mCallbackStreamId);
    192     if (res != OK) {
    193         ALOGE("%s: Unable to delete callback stream: %s (%d)",
    194                 __FUNCTION__, strerror(-res), res);
    195         return res;
    196     }
    197 
    198     {
    199         Mutex::Autolock l(mInputMutex);
    200 
    201         mCallbackHeap.clear();
    202         mCallbackWindow.clear();
    203         mCallbackConsumer.clear();
    204 
    205         mCallbackStreamId = NO_STREAM;
    206     }
    207     return OK;
    208 }
    209 
    210 int CallbackProcessor::getStreamId() const {
    211     Mutex::Autolock l(mInputMutex);
    212     return mCallbackStreamId;
    213 }
    214 
    215 void CallbackProcessor::dump(int /*fd*/, const Vector<String16>& /*args*/) const {
    216 }
    217 
    218 bool CallbackProcessor::threadLoop() {
    219     status_t res;
    220 
    221     {
    222         Mutex::Autolock l(mInputMutex);
    223         while (!mCallbackAvailable) {
    224             res = mCallbackAvailableSignal.waitRelative(mInputMutex,
    225                     kWaitDuration);
    226             if (res == TIMED_OUT) return true;
    227         }
    228         mCallbackAvailable = false;
    229     }
    230 
    231     do {
    232         sp<Camera2Client> client = mClient.promote();
    233         if (client == 0) {
    234             res = discardNewCallback();
    235         } else {
    236             res = processNewCallback(client);
    237         }
    238     } while (res == OK);
    239 
    240     return true;
    241 }
    242 
    243 status_t CallbackProcessor::discardNewCallback() {
    244     ATRACE_CALL();
    245     status_t res;
    246     CpuConsumer::LockedBuffer imgBuffer;
    247     res = mCallbackConsumer->lockNextBuffer(&imgBuffer);
    248     if (res != OK) {
    249         if (res != BAD_VALUE) {
    250             ALOGE("%s: Camera %d: Error receiving next callback buffer: "
    251                     "%s (%d)", __FUNCTION__, mId, strerror(-res), res);
    252         }
    253         return res;
    254     }
    255     mCallbackConsumer->unlockBuffer(imgBuffer);
    256     return OK;
    257 }
    258 
    259 status_t CallbackProcessor::processNewCallback(sp<Camera2Client> &client) {
    260     ATRACE_CALL();
    261     status_t res;
    262 
    263     sp<Camera2Heap> callbackHeap;
    264     bool useFlexibleYuv = false;
    265     int32_t previewFormat = 0;
    266     size_t heapIdx;
    267 
    268     {
    269         /* acquire SharedParameters before mMutex so we don't dead lock
    270             with Camera2Client code calling into StreamingProcessor */
    271         SharedParameters::Lock l(client->getParameters());
    272         Mutex::Autolock m(mInputMutex);
    273         CpuConsumer::LockedBuffer imgBuffer;
    274         if (mCallbackStreamId == NO_STREAM) {
    275             ALOGV("%s: Camera %d:No stream is available"
    276                     , __FUNCTION__, mId);
    277             return INVALID_OPERATION;
    278         }
    279 
    280         ALOGV("%s: Getting buffer", __FUNCTION__);
    281         res = mCallbackConsumer->lockNextBuffer(&imgBuffer);
    282         if (res != OK) {
    283             if (res != BAD_VALUE) {
    284                 ALOGE("%s: Camera %d: Error receiving next callback buffer: "
    285                         "%s (%d)", __FUNCTION__, mId, strerror(-res), res);
    286             }
    287             return res;
    288         }
    289         ALOGV("%s: Camera %d: Preview callback available", __FUNCTION__,
    290                 mId);
    291 
    292         if ( l.mParameters.state != Parameters::PREVIEW
    293                 && l.mParameters.state != Parameters::RECORD
    294                 && l.mParameters.state != Parameters::VIDEO_SNAPSHOT) {
    295             ALOGV("%s: Camera %d: No longer streaming",
    296                     __FUNCTION__, mId);
    297             mCallbackConsumer->unlockBuffer(imgBuffer);
    298             return OK;
    299         }
    300 
    301         if (! (l.mParameters.previewCallbackFlags &
    302                 CAMERA_FRAME_CALLBACK_FLAG_ENABLE_MASK) ) {
    303             ALOGV("%s: No longer enabled, dropping", __FUNCTION__);
    304             mCallbackConsumer->unlockBuffer(imgBuffer);
    305             return OK;
    306         }
    307         if ((l.mParameters.previewCallbackFlags &
    308                         CAMERA_FRAME_CALLBACK_FLAG_ONE_SHOT_MASK) &&
    309                 !l.mParameters.previewCallbackOneShot) {
    310             ALOGV("%s: One shot mode, already sent, dropping", __FUNCTION__);
    311             mCallbackConsumer->unlockBuffer(imgBuffer);
    312             return OK;
    313         }
    314 
    315         if (imgBuffer.width != static_cast<uint32_t>(l.mParameters.previewWidth) ||
    316                 imgBuffer.height != static_cast<uint32_t>(l.mParameters.previewHeight)) {
    317             ALOGW("%s: The preview size has changed to %d x %d from %d x %d, this buffer is"
    318                     " no longer valid, dropping",__FUNCTION__,
    319                     l.mParameters.previewWidth, l.mParameters.previewHeight,
    320                     imgBuffer.width, imgBuffer.height);
    321             mCallbackConsumer->unlockBuffer(imgBuffer);
    322             return OK;
    323         }
    324 
    325         previewFormat = l.mParameters.previewFormat;
    326         useFlexibleYuv = l.mParameters.fastInfo.useFlexibleYuv &&
    327                 (previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP ||
    328                  previewFormat == HAL_PIXEL_FORMAT_YV12);
    329 
    330         int32_t expectedFormat = useFlexibleYuv ?
    331                 HAL_PIXEL_FORMAT_YCbCr_420_888 : previewFormat;
    332 
    333         if (imgBuffer.format != expectedFormat) {
    334             ALOGE("%s: Camera %d: Unexpected format for callback: "
    335                     "0x%x, expected 0x%x", __FUNCTION__, mId,
    336                     imgBuffer.format, expectedFormat);
    337             mCallbackConsumer->unlockBuffer(imgBuffer);
    338             return INVALID_OPERATION;
    339         }
    340 
    341         // In one-shot mode, stop sending callbacks after the first one
    342         if (l.mParameters.previewCallbackFlags &
    343                 CAMERA_FRAME_CALLBACK_FLAG_ONE_SHOT_MASK) {
    344             ALOGV("%s: clearing oneshot", __FUNCTION__);
    345             l.mParameters.previewCallbackOneShot = false;
    346         }
    347 
    348         uint32_t destYStride = 0;
    349         uint32_t destCStride = 0;
    350         if (useFlexibleYuv) {
    351             if (previewFormat == HAL_PIXEL_FORMAT_YV12) {
    352                 // Strides must align to 16 for YV12
    353                 destYStride = ALIGN(imgBuffer.width, 16);
    354                 destCStride = ALIGN(destYStride / 2, 16);
    355             } else {
    356                 // No padding for NV21
    357                 ALOG_ASSERT(previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP,
    358                         "Unexpected preview format 0x%x", previewFormat);
    359                 destYStride = imgBuffer.width;
    360                 destCStride = destYStride / 2;
    361             }
    362         } else {
    363             destYStride = imgBuffer.stride;
    364             // don't care about cStride
    365         }
    366 
    367         size_t bufferSize = Camera2Client::calculateBufferSize(
    368                 imgBuffer.width, imgBuffer.height,
    369                 previewFormat, destYStride);
    370         size_t currentBufferSize = (mCallbackHeap == 0) ?
    371                 0 : (mCallbackHeap->mHeap->getSize() / kCallbackHeapCount);
    372         if (bufferSize != currentBufferSize) {
    373             mCallbackHeap.clear();
    374             mCallbackHeap = new Camera2Heap(bufferSize, kCallbackHeapCount,
    375                     "Camera2Client::CallbackHeap");
    376             if (mCallbackHeap->mHeap->getSize() == 0) {
    377                 ALOGE("%s: Camera %d: Unable to allocate memory for callbacks",
    378                         __FUNCTION__, mId);
    379                 mCallbackConsumer->unlockBuffer(imgBuffer);
    380                 return INVALID_OPERATION;
    381             }
    382 
    383             mCallbackHeapHead = 0;
    384             mCallbackHeapFree = kCallbackHeapCount;
    385         }
    386 
    387         if (mCallbackHeapFree == 0) {
    388             ALOGE("%s: Camera %d: No free callback buffers, dropping frame",
    389                     __FUNCTION__, mId);
    390             mCallbackConsumer->unlockBuffer(imgBuffer);
    391             return OK;
    392         }
    393 
    394         heapIdx = mCallbackHeapHead;
    395 
    396         mCallbackHeapHead = (mCallbackHeapHead + 1) & kCallbackHeapCount;
    397         mCallbackHeapFree--;
    398 
    399         // TODO: Get rid of this copy by passing the gralloc queue all the way
    400         // to app
    401 
    402         ssize_t offset;
    403         size_t size;
    404         sp<IMemoryHeap> heap =
    405                 mCallbackHeap->mBuffers[heapIdx]->getMemory(&offset,
    406                         &size);
    407         uint8_t *data = (uint8_t*)heap->getBase() + offset;
    408 
    409         if (!useFlexibleYuv) {
    410             // Can just memcpy when HAL format matches API format
    411             memcpy(data, imgBuffer.data, bufferSize);
    412         } else {
    413             res = convertFromFlexibleYuv(previewFormat, data, imgBuffer,
    414                     destYStride, destCStride);
    415             if (res != OK) {
    416                 ALOGE("%s: Camera %d: Can't convert between 0x%x and 0x%x formats!",
    417                         __FUNCTION__, mId, imgBuffer.format, previewFormat);
    418                 mCallbackConsumer->unlockBuffer(imgBuffer);
    419                 return BAD_VALUE;
    420             }
    421         }
    422 
    423         ALOGV("%s: Freeing buffer", __FUNCTION__);
    424         mCallbackConsumer->unlockBuffer(imgBuffer);
    425 
    426         // mCallbackHeap may get freed up once input mutex is released
    427         callbackHeap = mCallbackHeap;
    428     }
    429 
    430     // Call outside parameter lock to allow re-entrancy from notification
    431     {
    432         Camera2Client::SharedCameraCallbacks::Lock
    433             l(client->mSharedCameraCallbacks);
    434         if (l.mRemoteCallback != 0) {
    435             ALOGV("%s: Camera %d: Invoking client data callback",
    436                     __FUNCTION__, mId);
    437             l.mRemoteCallback->dataCallback(CAMERA_MSG_PREVIEW_FRAME,
    438                     callbackHeap->mBuffers[heapIdx], NULL);
    439         }
    440     }
    441 
    442     // Only increment free if we're still using the same heap
    443     mCallbackHeapFree++;
    444 
    445     ALOGV("%s: exit", __FUNCTION__);
    446 
    447     return OK;
    448 }
    449 
    450 status_t CallbackProcessor::convertFromFlexibleYuv(int32_t previewFormat,
    451         uint8_t *dst,
    452         const CpuConsumer::LockedBuffer &src,
    453         uint32_t dstYStride,
    454         uint32_t dstCStride) const {
    455 
    456     if (previewFormat != HAL_PIXEL_FORMAT_YCrCb_420_SP &&
    457             previewFormat != HAL_PIXEL_FORMAT_YV12) {
    458         ALOGE("%s: Camera %d: Unexpected preview format when using "
    459                 "flexible YUV: 0x%x", __FUNCTION__, mId, previewFormat);
    460         return INVALID_OPERATION;
    461     }
    462 
    463     // Copy Y plane, adjusting for stride
    464     const uint8_t *ySrc = src.data;
    465     uint8_t *yDst = dst;
    466     for (size_t row = 0; row < src.height; row++) {
    467         memcpy(yDst, ySrc, src.width);
    468         ySrc += src.stride;
    469         yDst += dstYStride;
    470     }
    471 
    472     // Copy/swizzle chroma planes, 4:2:0 subsampling
    473     const uint8_t *cbSrc = src.dataCb;
    474     const uint8_t *crSrc = src.dataCr;
    475     size_t chromaHeight = src.height / 2;
    476     size_t chromaWidth = src.width / 2;
    477     ssize_t chromaGap = src.chromaStride -
    478             (chromaWidth * src.chromaStep);
    479     size_t dstChromaGap = dstCStride - chromaWidth;
    480 
    481     if (previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) {
    482         // Flexible YUV chroma to NV21 chroma
    483         uint8_t *crcbDst = yDst;
    484         // Check for shortcuts
    485         if (cbSrc == crSrc + 1 && src.chromaStep == 2) {
    486             ALOGV("%s: Fast NV21->NV21", __FUNCTION__);
    487             // Source has semiplanar CrCb chroma layout, can copy by rows
    488             for (size_t row = 0; row < chromaHeight; row++) {
    489                 memcpy(crcbDst, crSrc, src.width);
    490                 crcbDst += src.width;
    491                 crSrc += src.chromaStride;
    492             }
    493         } else {
    494             ALOGV("%s: Generic->NV21", __FUNCTION__);
    495             // Generic copy, always works but not very efficient
    496             for (size_t row = 0; row < chromaHeight; row++) {
    497                 for (size_t col = 0; col < chromaWidth; col++) {
    498                     *(crcbDst++) = *crSrc;
    499                     *(crcbDst++) = *cbSrc;
    500                     crSrc += src.chromaStep;
    501                     cbSrc += src.chromaStep;
    502                 }
    503                 crSrc += chromaGap;
    504                 cbSrc += chromaGap;
    505             }
    506         }
    507     } else {
    508         // flexible YUV chroma to YV12 chroma
    509         ALOG_ASSERT(previewFormat == HAL_PIXEL_FORMAT_YV12,
    510                 "Unexpected preview format 0x%x", previewFormat);
    511         uint8_t *crDst = yDst;
    512         uint8_t *cbDst = yDst + chromaHeight * dstCStride;
    513         if (src.chromaStep == 1) {
    514             ALOGV("%s: Fast YV12->YV12", __FUNCTION__);
    515             // Source has planar chroma layout, can copy by row
    516             for (size_t row = 0; row < chromaHeight; row++) {
    517                 memcpy(crDst, crSrc, chromaWidth);
    518                 crDst += dstCStride;
    519                 crSrc += src.chromaStride;
    520             }
    521             for (size_t row = 0; row < chromaHeight; row++) {
    522                 memcpy(cbDst, cbSrc, chromaWidth);
    523                 cbDst += dstCStride;
    524                 cbSrc += src.chromaStride;
    525             }
    526         } else {
    527             ALOGV("%s: Generic->YV12", __FUNCTION__);
    528             // Generic copy, always works but not very efficient
    529             for (size_t row = 0; row < chromaHeight; row++) {
    530                 for (size_t col = 0; col < chromaWidth; col++) {
    531                     *(crDst++) = *crSrc;
    532                     *(cbDst++) = *cbSrc;
    533                     crSrc += src.chromaStep;
    534                     cbSrc += src.chromaStep;
    535                 }
    536                 crSrc += chromaGap;
    537                 cbSrc += chromaGap;
    538                 crDst += dstChromaGap;
    539                 cbDst += dstChromaGap;
    540             }
    541         }
    542     }
    543 
    544     return OK;
    545 }
    546 
    547 }; // namespace camera2
    548 }; // namespace android
    549