Home | History | Annotate | Download | only in client2
      1 /*
      2  * Copyright (C) 2012 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #define LOG_TAG "Camera2-ZslProcessor"
     18 #define ATRACE_TAG ATRACE_TAG_CAMERA
     19 //#define LOG_NDEBUG 0
     20 //#define LOG_NNDEBUG 0
     21 
     22 #ifdef LOG_NNDEBUG
     23 #define ALOGVV(...) ALOGV(__VA_ARGS__)
     24 #else
     25 #define ALOGVV(...) ((void)0)
     26 #endif
     27 
     28 #include <inttypes.h>
     29 
     30 #include <utils/Log.h>
     31 #include <utils/Trace.h>
     32 #include <gui/Surface.h>
     33 
     34 #include "common/CameraDeviceBase.h"
     35 #include "api1/Camera2Client.h"
     36 #include "api1/client2/CaptureSequencer.h"
     37 #include "api1/client2/ZslProcessor.h"
     38 
     39 namespace android {
     40 namespace camera2 {
     41 
     42 ZslProcessor::ZslProcessor(
     43     sp<Camera2Client> client,
     44     wp<CaptureSequencer> sequencer):
     45         Thread(false),
     46         mState(RUNNING),
     47         mClient(client),
     48         mDevice(client->getCameraDevice()),
     49         mSequencer(sequencer),
     50         mId(client->getCameraId()),
     51         mDeleted(false),
     52         mZslBufferAvailable(false),
     53         mZslStreamId(NO_STREAM),
     54         mZslReprocessStreamId(NO_STREAM),
     55         mFrameListHead(0),
     56         mZslQueueHead(0),
     57         mZslQueueTail(0) {
     58     mZslQueue.insertAt(0, kZslBufferDepth);
     59     mFrameList.insertAt(0, kFrameListDepth);
     60     sp<CaptureSequencer> captureSequencer = mSequencer.promote();
     61     if (captureSequencer != 0) captureSequencer->setZslProcessor(this);
     62 }
     63 
     64 ZslProcessor::~ZslProcessor() {
     65     ALOGV("%s: Exit", __FUNCTION__);
     66     disconnect();
     67 }
     68 
     69 void ZslProcessor::onFrameAvailable() {
     70     Mutex::Autolock l(mInputMutex);
     71     if (!mZslBufferAvailable) {
     72         mZslBufferAvailable = true;
     73         mZslBufferAvailableSignal.signal();
     74     }
     75 }
     76 
     77 void ZslProcessor::onResultAvailable(const CaptureResult &result) {
     78     ATRACE_CALL();
     79     ALOGV("%s:", __FUNCTION__);
     80     Mutex::Autolock l(mInputMutex);
     81     camera_metadata_ro_entry_t entry;
     82     entry = result.mMetadata.find(ANDROID_SENSOR_TIMESTAMP);
     83     nsecs_t timestamp = entry.data.i64[0];
     84     (void)timestamp;
     85     ALOGVV("Got preview frame for timestamp %" PRId64, timestamp);
     86 
     87     if (mState != RUNNING) return;
     88 
     89     mFrameList.editItemAt(mFrameListHead) = result.mMetadata;
     90     mFrameListHead = (mFrameListHead + 1) % kFrameListDepth;
     91 
     92     findMatchesLocked();
     93 }
     94 
     95 void ZslProcessor::onBufferReleased(buffer_handle_t *handle) {
     96     Mutex::Autolock l(mInputMutex);
     97 
     98     // Verify that the buffer is in our queue
     99     size_t i = 0;
    100     for (; i < mZslQueue.size(); i++) {
    101         if (&(mZslQueue[i].buffer.mGraphicBuffer->handle) == handle) break;
    102     }
    103     if (i == mZslQueue.size()) {
    104         ALOGW("%s: Released buffer %p not found in queue",
    105                 __FUNCTION__, handle);
    106     }
    107 
    108     // Erase entire ZSL queue since we've now completed the capture and preview
    109     // is stopped.
    110     clearZslQueueLocked();
    111 
    112     mState = RUNNING;
    113 }
    114 
    115 status_t ZslProcessor::updateStream(const Parameters &params) {
    116     ATRACE_CALL();
    117     ALOGV("%s: Configuring ZSL streams", __FUNCTION__);
    118     status_t res;
    119 
    120     Mutex::Autolock l(mInputMutex);
    121 
    122     sp<Camera2Client> client = mClient.promote();
    123     if (client == 0) {
    124         ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
    125         return INVALID_OPERATION;
    126     }
    127     sp<CameraDeviceBase> device = mDevice.promote();
    128     if (device == 0) {
    129         ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
    130         return INVALID_OPERATION;
    131     }
    132 
    133     if (mZslConsumer == 0) {
    134         // Create CPU buffer queue endpoint
    135         sp<IGraphicBufferProducer> producer;
    136         sp<IGraphicBufferConsumer> consumer;
    137         BufferQueue::createBufferQueue(&producer, &consumer);
    138         mZslConsumer = new BufferItemConsumer(consumer,
    139             GRALLOC_USAGE_HW_CAMERA_ZSL,
    140             kZslBufferDepth);
    141         mZslConsumer->setFrameAvailableListener(this);
    142         mZslConsumer->setName(String8("Camera2Client::ZslConsumer"));
    143         mZslWindow = new Surface(producer);
    144     }
    145 
    146     if (mZslStreamId != NO_STREAM) {
    147         // Check if stream parameters have to change
    148         uint32_t currentWidth, currentHeight;
    149         res = device->getStreamInfo(mZslStreamId,
    150                 &currentWidth, &currentHeight, 0);
    151         if (res != OK) {
    152             ALOGE("%s: Camera %d: Error querying capture output stream info: "
    153                     "%s (%d)", __FUNCTION__,
    154                     mId, strerror(-res), res);
    155             return res;
    156         }
    157         if (currentWidth != (uint32_t)params.fastInfo.arrayWidth ||
    158                 currentHeight != (uint32_t)params.fastInfo.arrayHeight) {
    159             res = device->deleteReprocessStream(mZslReprocessStreamId);
    160             if (res != OK) {
    161                 ALOGE("%s: Camera %d: Unable to delete old reprocess stream "
    162                         "for ZSL: %s (%d)", __FUNCTION__,
    163                         mId, strerror(-res), res);
    164                 return res;
    165             }
    166             ALOGV("%s: Camera %d: Deleting stream %d since the buffer dimensions changed",
    167                 __FUNCTION__, mId, mZslStreamId);
    168             res = device->deleteStream(mZslStreamId);
    169             if (res != OK) {
    170                 ALOGE("%s: Camera %d: Unable to delete old output stream "
    171                         "for ZSL: %s (%d)", __FUNCTION__,
    172                         mId, strerror(-res), res);
    173                 return res;
    174             }
    175             mZslStreamId = NO_STREAM;
    176         }
    177     }
    178 
    179     mDeleted = false;
    180 
    181     if (mZslStreamId == NO_STREAM) {
    182         // Create stream for HAL production
    183         // TODO: Sort out better way to select resolution for ZSL
    184         int streamType = params.quirks.useZslFormat ?
    185                 (int)CAMERA2_HAL_PIXEL_FORMAT_ZSL :
    186                 (int)HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
    187         res = device->createStream(mZslWindow,
    188                 params.fastInfo.arrayWidth, params.fastInfo.arrayHeight,
    189                 streamType, &mZslStreamId);
    190         if (res != OK) {
    191             ALOGE("%s: Camera %d: Can't create output stream for ZSL: "
    192                     "%s (%d)", __FUNCTION__, mId,
    193                     strerror(-res), res);
    194             return res;
    195         }
    196         res = device->createReprocessStreamFromStream(mZslStreamId,
    197                 &mZslReprocessStreamId);
    198         if (res != OK) {
    199             ALOGE("%s: Camera %d: Can't create reprocess stream for ZSL: "
    200                     "%s (%d)", __FUNCTION__, mId,
    201                     strerror(-res), res);
    202             return res;
    203         }
    204     }
    205     client->registerFrameListener(Camera2Client::kPreviewRequestIdStart,
    206             Camera2Client::kPreviewRequestIdEnd,
    207             this,
    208             /*sendPartials*/false);
    209 
    210     return OK;
    211 }
    212 
    213 status_t ZslProcessor::deleteStream() {
    214     ATRACE_CALL();
    215     Mutex::Autolock l(mInputMutex);
    216     // WAR(b/15408128): do not delete stream unless client is being disconnected.
    217     mDeleted = true;
    218     return OK;
    219 }
    220 
    221 status_t ZslProcessor::disconnect() {
    222     ATRACE_CALL();
    223     status_t res;
    224 
    225     Mutex::Autolock l(mInputMutex);
    226 
    227     if (mZslStreamId != NO_STREAM) {
    228         sp<CameraDeviceBase> device = mDevice.promote();
    229         if (device == 0) {
    230             ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
    231             return INVALID_OPERATION;
    232         }
    233 
    234         clearZslQueueLocked();
    235 
    236         res = device->deleteReprocessStream(mZslReprocessStreamId);
    237         if (res != OK) {
    238             ALOGE("%s: Camera %d: Cannot delete ZSL reprocessing stream %d: "
    239                     "%s (%d)", __FUNCTION__, mId,
    240                     mZslReprocessStreamId, strerror(-res), res);
    241             return res;
    242         }
    243 
    244         mZslReprocessStreamId = NO_STREAM;
    245         res = device->deleteStream(mZslStreamId);
    246         if (res != OK) {
    247             ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: "
    248                     "%s (%d)", __FUNCTION__, mId,
    249                     mZslStreamId, strerror(-res), res);
    250             return res;
    251         }
    252 
    253         mZslWindow.clear();
    254         mZslConsumer.clear();
    255 
    256         mZslStreamId = NO_STREAM;
    257     }
    258     return OK;
    259 }
    260 
    261 int ZslProcessor::getStreamId() const {
    262     Mutex::Autolock l(mInputMutex);
    263     return mZslStreamId;
    264 }
    265 
    266 status_t ZslProcessor::pushToReprocess(int32_t requestId) {
    267     ALOGV("%s: Send in reprocess request with id %d",
    268             __FUNCTION__, requestId);
    269     Mutex::Autolock l(mInputMutex);
    270     status_t res;
    271     sp<Camera2Client> client = mClient.promote();
    272 
    273     if (client == 0) {
    274         ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
    275         return INVALID_OPERATION;
    276     }
    277 
    278     IF_ALOGV() {
    279         dumpZslQueue(-1);
    280     }
    281 
    282     if (mZslQueueTail != mZslQueueHead) {
    283         CameraMetadata request;
    284         size_t index = mZslQueueTail;
    285         while (index != mZslQueueHead) {
    286             if (!mZslQueue[index].frame.isEmpty()) {
    287                 request = mZslQueue[index].frame;
    288                 break;
    289             }
    290             index = (index + 1) % kZslBufferDepth;
    291         }
    292         if (index == mZslQueueHead) {
    293             ALOGV("%s: ZSL queue has no valid frames to send yet.",
    294                   __FUNCTION__);
    295             return NOT_ENOUGH_DATA;
    296         }
    297         // Verify that the frame is reasonable for reprocessing
    298 
    299         camera_metadata_entry_t entry;
    300         entry = request.find(ANDROID_CONTROL_AE_STATE);
    301         if (entry.count == 0) {
    302             ALOGE("%s: ZSL queue frame has no AE state field!",
    303                     __FUNCTION__);
    304             return BAD_VALUE;
    305         }
    306         if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED &&
    307                 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) {
    308             ALOGV("%s: ZSL queue frame AE state is %d, need full capture",
    309                     __FUNCTION__, entry.data.u8[0]);
    310             return NOT_ENOUGH_DATA;
    311         }
    312 
    313         buffer_handle_t *handle =
    314             &(mZslQueue[index].buffer.mGraphicBuffer->handle);
    315 
    316         uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS;
    317         res = request.update(ANDROID_REQUEST_TYPE,
    318                 &requestType, 1);
    319         int32_t inputStreams[1] =
    320                 { mZslReprocessStreamId };
    321         if (res == OK) request.update(ANDROID_REQUEST_INPUT_STREAMS,
    322                 inputStreams, 1);
    323         int32_t outputStreams[1] =
    324                 { client->getCaptureStreamId() };
    325         if (res == OK) request.update(ANDROID_REQUEST_OUTPUT_STREAMS,
    326                 outputStreams, 1);
    327         res = request.update(ANDROID_REQUEST_ID,
    328                 &requestId, 1);
    329 
    330         if (res != OK ) {
    331             ALOGE("%s: Unable to update frame to a reprocess request", __FUNCTION__);
    332             return INVALID_OPERATION;
    333         }
    334 
    335         res = client->stopStream();
    336         if (res != OK) {
    337             ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: "
    338                 "%s (%d)",
    339                 __FUNCTION__, mId, strerror(-res), res);
    340             return INVALID_OPERATION;
    341         }
    342         // TODO: have push-and-clear be atomic
    343         res = client->getCameraDevice()->pushReprocessBuffer(mZslReprocessStreamId,
    344                 handle, this);
    345         if (res != OK) {
    346             ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)",
    347                     __FUNCTION__, strerror(-res), res);
    348             return res;
    349         }
    350 
    351         // Update JPEG settings
    352         {
    353             SharedParameters::Lock l(client->getParameters());
    354             res = l.mParameters.updateRequestJpeg(&request);
    355             if (res != OK) {
    356                 ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL "
    357                         "capture request: %s (%d)", __FUNCTION__,
    358                         mId,
    359                         strerror(-res), res);
    360                 return res;
    361             }
    362         }
    363 
    364         mLatestCapturedRequest = request;
    365         res = client->getCameraDevice()->capture(request);
    366         if (res != OK ) {
    367             ALOGE("%s: Unable to send ZSL reprocess request to capture: %s (%d)",
    368                     __FUNCTION__, strerror(-res), res);
    369             return res;
    370         }
    371 
    372         mState = LOCKED;
    373     } else {
    374         ALOGV("%s: No ZSL buffers yet", __FUNCTION__);
    375         return NOT_ENOUGH_DATA;
    376     }
    377     return OK;
    378 }
    379 
    380 status_t ZslProcessor::clearZslQueue() {
    381     Mutex::Autolock l(mInputMutex);
    382     // If in middle of capture, can't clear out queue
    383     if (mState == LOCKED) return OK;
    384 
    385     return clearZslQueueLocked();
    386 }
    387 
    388 status_t ZslProcessor::clearZslQueueLocked() {
    389     for (size_t i = 0; i < mZslQueue.size(); i++) {
    390         if (mZslQueue[i].buffer.mTimestamp != 0) {
    391             mZslConsumer->releaseBuffer(mZslQueue[i].buffer);
    392         }
    393         mZslQueue.replaceAt(i);
    394     }
    395     mZslQueueHead = 0;
    396     mZslQueueTail = 0;
    397     return OK;
    398 }
    399 
    400 void ZslProcessor::dump(int fd, const Vector<String16>& /*args*/) const {
    401     Mutex::Autolock l(mInputMutex);
    402     if (!mLatestCapturedRequest.isEmpty()) {
    403         String8 result("    Latest ZSL capture request:\n");
    404         write(fd, result.string(), result.size());
    405         mLatestCapturedRequest.dump(fd, 2, 6);
    406     } else {
    407         String8 result("    Latest ZSL capture request: none yet\n");
    408         write(fd, result.string(), result.size());
    409     }
    410     dumpZslQueue(fd);
    411 }
    412 
    413 bool ZslProcessor::threadLoop() {
    414     status_t res;
    415 
    416     {
    417         Mutex::Autolock l(mInputMutex);
    418         while (!mZslBufferAvailable) {
    419             res = mZslBufferAvailableSignal.waitRelative(mInputMutex,
    420                     kWaitDuration);
    421             if (res == TIMED_OUT) return true;
    422         }
    423         mZslBufferAvailable = false;
    424     }
    425 
    426     do {
    427         res = processNewZslBuffer();
    428     } while (res == OK);
    429 
    430     return true;
    431 }
    432 
    433 status_t ZslProcessor::processNewZslBuffer() {
    434     ATRACE_CALL();
    435     status_t res;
    436     sp<BufferItemConsumer> zslConsumer;
    437     {
    438         Mutex::Autolock l(mInputMutex);
    439         if (mZslConsumer == 0) return OK;
    440         zslConsumer = mZslConsumer;
    441     }
    442     ALOGVV("Trying to get next buffer");
    443     BufferItemConsumer::BufferItem item;
    444     res = zslConsumer->acquireBuffer(&item, 0);
    445     if (res != OK) {
    446         if (res != BufferItemConsumer::NO_BUFFER_AVAILABLE) {
    447             ALOGE("%s: Camera %d: Error receiving ZSL image buffer: "
    448                     "%s (%d)", __FUNCTION__,
    449                     mId, strerror(-res), res);
    450         } else {
    451             ALOGVV("  No buffer");
    452         }
    453         return res;
    454     }
    455 
    456     Mutex::Autolock l(mInputMutex);
    457 
    458     if (mState == LOCKED) {
    459         ALOGVV("In capture, discarding new ZSL buffers");
    460         zslConsumer->releaseBuffer(item);
    461         return OK;
    462     }
    463 
    464     ALOGVV("Got ZSL buffer: head: %d, tail: %d", mZslQueueHead, mZslQueueTail);
    465 
    466     if ( (mZslQueueHead + 1) % kZslBufferDepth == mZslQueueTail) {
    467         ALOGVV("Releasing oldest buffer");
    468         zslConsumer->releaseBuffer(mZslQueue[mZslQueueTail].buffer);
    469         mZslQueue.replaceAt(mZslQueueTail);
    470         mZslQueueTail = (mZslQueueTail + 1) % kZslBufferDepth;
    471     }
    472 
    473     ZslPair &queueHead = mZslQueue.editItemAt(mZslQueueHead);
    474 
    475     queueHead.buffer = item;
    476     queueHead.frame.release();
    477 
    478     mZslQueueHead = (mZslQueueHead + 1) % kZslBufferDepth;
    479 
    480     ALOGVV("  Acquired buffer, timestamp %" PRId64, queueHead.buffer.mTimestamp);
    481 
    482     findMatchesLocked();
    483 
    484     return OK;
    485 }
    486 
    487 void ZslProcessor::findMatchesLocked() {
    488     ALOGVV("Scanning");
    489     for (size_t i = 0; i < mZslQueue.size(); i++) {
    490         ZslPair &queueEntry = mZslQueue.editItemAt(i);
    491         nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
    492         IF_ALOGV() {
    493             camera_metadata_entry_t entry;
    494             nsecs_t frameTimestamp = 0;
    495             if (!queueEntry.frame.isEmpty()) {
    496                 entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
    497                 frameTimestamp = entry.data.i64[0];
    498             }
    499             ALOGVV("   %d: b: %" PRId64 "\tf: %" PRId64, i,
    500                     bufferTimestamp, frameTimestamp );
    501         }
    502         if (queueEntry.frame.isEmpty() && bufferTimestamp != 0) {
    503             // Have buffer, no matching frame. Look for one
    504             for (size_t j = 0; j < mFrameList.size(); j++) {
    505                 bool match = false;
    506                 CameraMetadata &frame = mFrameList.editItemAt(j);
    507                 if (!frame.isEmpty()) {
    508                     camera_metadata_entry_t entry;
    509                     entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
    510                     if (entry.count == 0) {
    511                         ALOGE("%s: Can't find timestamp in frame!",
    512                                 __FUNCTION__);
    513                         continue;
    514                     }
    515                     nsecs_t frameTimestamp = entry.data.i64[0];
    516                     if (bufferTimestamp == frameTimestamp) {
    517                         ALOGVV("%s: Found match %" PRId64, __FUNCTION__,
    518                                 frameTimestamp);
    519                         match = true;
    520                     } else {
    521                         int64_t delta = abs(bufferTimestamp - frameTimestamp);
    522                         if ( delta < 1000000) {
    523                             ALOGVV("%s: Found close match %" PRId64 " (delta %" PRId64 ")",
    524                                     __FUNCTION__, bufferTimestamp, delta);
    525                             match = true;
    526                         }
    527                     }
    528                 }
    529                 if (match) {
    530                     queueEntry.frame.acquire(frame);
    531                     break;
    532                 }
    533             }
    534         }
    535     }
    536 }
    537 
    538 void ZslProcessor::dumpZslQueue(int fd) const {
    539     String8 header("ZSL queue contents:");
    540     String8 indent("    ");
    541     ALOGV("%s", header.string());
    542     if (fd != -1) {
    543         header = indent + header + "\n";
    544         write(fd, header.string(), header.size());
    545     }
    546     for (size_t i = 0; i < mZslQueue.size(); i++) {
    547         const ZslPair &queueEntry = mZslQueue[i];
    548         nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
    549         camera_metadata_ro_entry_t entry;
    550         nsecs_t frameTimestamp = 0;
    551         int frameAeState = -1;
    552         if (!queueEntry.frame.isEmpty()) {
    553             entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
    554             if (entry.count > 0) frameTimestamp = entry.data.i64[0];
    555             entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE);
    556             if (entry.count > 0) frameAeState = entry.data.u8[0];
    557         }
    558         String8 result =
    559                 String8::format("   %zu: b: %" PRId64 "\tf: %" PRId64 ", AE state: %d", i,
    560                         bufferTimestamp, frameTimestamp, frameAeState);
    561         ALOGV("%s", result.string());
    562         if (fd != -1) {
    563             result = indent + result + "\n";
    564             write(fd, result.string(), result.size());
    565         }
    566 
    567     }
    568 }
    569 
    570 }; // namespace camera2
    571 }; // namespace android
    572