Home | History | Annotate | Download | only in client2 
      1 /*
      2  * Copyright (C) 2013 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #define LOG_TAG "Camera2-ZslProcessor3"
     18 #define ATRACE_TAG ATRACE_TAG_CAMERA
     19 //#define LOG_NDEBUG 0
     20 //#define LOG_NNDEBUG 0
     21 
     22 #ifdef LOG_NNDEBUG
     23 #define ALOGVV(...) ALOGV(__VA_ARGS__)
     24 #else
     25 #define ALOGVV(...) ((void)0)
     26 #endif
     27 
     28 #include <utils/Log.h>
     29 #include <utils/Trace.h>
     30 #include <gui/Surface.h>
     31 
     32 #include "common/CameraDeviceBase.h"
     33 #include "api1/Camera2Client.h"
     34 #include "api1/client2/CaptureSequencer.h"
     35 #include "api1/client2/ZslProcessor3.h"
     36 #include "device3/Camera3Device.h"
     37 
     38 namespace android {
     39 namespace camera2 {
     40 
     41 ZslProcessor3::ZslProcessor3(
     42     sp<Camera2Client> client,
     43     wp<CaptureSequencer> sequencer):
     44         Thread(false),
     45         mState(RUNNING),
     46         mClient(client),
     47         mSequencer(sequencer),
     48         mId(client->getCameraId()),
     49         mZslStreamId(NO_STREAM),
     50         mFrameListHead(0),
     51         mZslQueueHead(0),
     52         mZslQueueTail(0) {
     53     mZslQueue.insertAt(0, kZslBufferDepth);
     54     mFrameList.insertAt(0, kFrameListDepth);
     55     sp<CaptureSequencer> captureSequencer = mSequencer.promote();
     56     if (captureSequencer != 0) captureSequencer->setZslProcessor(this);
     57 }
     58 
     59 ZslProcessor3::~ZslProcessor3() {
     60     ALOGV("%s: Exit", __FUNCTION__);
     61     deleteStream();
     62 }
     63 
     64 void ZslProcessor3::onFrameAvailable(int32_t /*requestId*/,
     65                                      const CameraMetadata &frame) {
     66     Mutex::Autolock l(mInputMutex);
     67     camera_metadata_ro_entry_t entry;
     68     entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
     69     nsecs_t timestamp = entry.data.i64[0];
     70     (void)timestamp;
     71     ALOGVV("Got preview metadata for timestamp %lld", timestamp);
     72 
     73     if (mState != RUNNING) return;
     74 
     75     mFrameList.editItemAt(mFrameListHead) = frame;
     76     mFrameListHead = (mFrameListHead + 1) % kFrameListDepth;
     77 }
     78 
     79 status_t ZslProcessor3::updateStream(const Parameters &params) {
     80     ATRACE_CALL();
     81     ALOGV("%s: Configuring ZSL streams", __FUNCTION__);
     82     status_t res;
     83 
     84     Mutex::Autolock l(mInputMutex);
     85 
     86     sp<Camera2Client> client = mClient.promote();
     87     if (client == 0) {
     88         ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
     89         return INVALID_OPERATION;
     90     }
     91     sp<Camera3Device> device =
     92         static_cast<Camera3Device*>(client->getCameraDevice().get());
     93     if (device == 0) {
     94         ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
     95         return INVALID_OPERATION;
     96     }
     97 
     98     if (mZslStreamId != NO_STREAM) {
     99         // Check if stream parameters have to change
    100         uint32_t currentWidth, currentHeight;
    101         res = device->getStreamInfo(mZslStreamId,
    102                 &currentWidth, &currentHeight, 0);
    103         if (res != OK) {
    104             ALOGE("%s: Camera %d: Error querying capture output stream info: "
    105                     "%s (%d)", __FUNCTION__,
    106                     client->getCameraId(), strerror(-res), res);
    107             return res;
    108         }
    109         if (currentWidth != (uint32_t)params.fastInfo.arrayWidth ||
    110                 currentHeight != (uint32_t)params.fastInfo.arrayHeight) {
    111             ALOGV("%s: Camera %d: Deleting stream %d since the buffer "
    112                   "dimensions changed",
    113                 __FUNCTION__, client->getCameraId(), mZslStreamId);
    114             res = device->deleteStream(mZslStreamId);
    115             if (res == -EBUSY) {
    116                 ALOGV("%s: Camera %d: Device is busy, call updateStream again "
    117                       " after it becomes idle", __FUNCTION__, mId);
    118                 return res;
    119             } else if(res != OK) {
    120                 ALOGE("%s: Camera %d: Unable to delete old output stream "
    121                         "for ZSL: %s (%d)", __FUNCTION__,
    122                         client->getCameraId(), strerror(-res), res);
    123                 return res;
    124             }
    125             mZslStreamId = NO_STREAM;
    126         }
    127     }
    128 
    129     if (mZslStreamId == NO_STREAM) {
    130         // Create stream for HAL production
    131         // TODO: Sort out better way to select resolution for ZSL
    132 
    133         // Note that format specified internally in Camera3ZslStream
    134         res = device->createZslStream(
    135                 params.fastInfo.arrayWidth, params.fastInfo.arrayHeight,
    136                 kZslBufferDepth,
    137                 &mZslStreamId,
    138                 &mZslStream);
    139         if (res != OK) {
    140             ALOGE("%s: Camera %d: Can't create ZSL stream: "
    141                     "%s (%d)", __FUNCTION__, client->getCameraId(),
    142                     strerror(-res), res);
    143             return res;
    144         }
    145     }
    146     client->registerFrameListener(Camera2Client::kPreviewRequestIdStart,
    147             Camera2Client::kPreviewRequestIdEnd,
    148             this);
    149 
    150     return OK;
    151 }
    152 
    153 status_t ZslProcessor3::deleteStream() {
    154     ATRACE_CALL();
    155     status_t res;
    156 
    157     Mutex::Autolock l(mInputMutex);
    158 
    159     if (mZslStreamId != NO_STREAM) {
    160         sp<Camera2Client> client = mClient.promote();
    161         if (client == 0) {
    162             ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
    163             return INVALID_OPERATION;
    164         }
    165 
    166         sp<Camera3Device> device =
    167             reinterpret_cast<Camera3Device*>(client->getCameraDevice().get());
    168         if (device == 0) {
    169             ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
    170             return INVALID_OPERATION;
    171         }
    172 
    173         res = device->deleteStream(mZslStreamId);
    174         if (res != OK) {
    175             ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: "
    176                     "%s (%d)", __FUNCTION__, client->getCameraId(),
    177                     mZslStreamId, strerror(-res), res);
    178             return res;
    179         }
    180 
    181         mZslStreamId = NO_STREAM;
    182     }
    183     return OK;
    184 }
    185 
    186 int ZslProcessor3::getStreamId() const {
    187     Mutex::Autolock l(mInputMutex);
    188     return mZslStreamId;
    189 }
    190 
    191 status_t ZslProcessor3::pushToReprocess(int32_t requestId) {
    192     ALOGV("%s: Send in reprocess request with id %d",
    193             __FUNCTION__, requestId);
    194     Mutex::Autolock l(mInputMutex);
    195     status_t res;
    196     sp<Camera2Client> client = mClient.promote();
    197 
    198     if (client == 0) {
    199         ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
    200         return INVALID_OPERATION;
    201     }
    202 
    203     IF_ALOGV() {
    204         dumpZslQueue(-1);
    205     }
    206 
    207     size_t metadataIdx;
    208     nsecs_t candidateTimestamp = getCandidateTimestampLocked(&metadataIdx);
    209 
    210     if (candidateTimestamp == -1) {
    211         ALOGE("%s: Could not find good candidate for ZSL reprocessing",
    212               __FUNCTION__);
    213         return NOT_ENOUGH_DATA;
    214     }
    215 
    216     res = mZslStream->enqueueInputBufferByTimestamp(candidateTimestamp,
    217                                                     /*actualTimestamp*/NULL);
    218 
    219     if (res == mZslStream->NO_BUFFER_AVAILABLE) {
    220         ALOGV("%s: No ZSL buffers yet", __FUNCTION__);
    221         return NOT_ENOUGH_DATA;
    222     } else if (res != OK) {
    223         ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)",
    224                 __FUNCTION__, strerror(-res), res);
    225         return res;
    226     }
    227 
    228     {
    229         CameraMetadata request = mFrameList[metadataIdx];
    230 
    231         // Verify that the frame is reasonable for reprocessing
    232 
    233         camera_metadata_entry_t entry;
    234         entry = request.find(ANDROID_CONTROL_AE_STATE);
    235         if (entry.count == 0) {
    236             ALOGE("%s: ZSL queue frame has no AE state field!",
    237                     __FUNCTION__);
    238             return BAD_VALUE;
    239         }
    240         if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED &&
    241                 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) {
    242             ALOGV("%s: ZSL queue frame AE state is %d, need full capture",
    243                     __FUNCTION__, entry.data.u8[0]);
    244             return NOT_ENOUGH_DATA;
    245         }
    246 
    247         uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS;
    248         res = request.update(ANDROID_REQUEST_TYPE,
    249                 &requestType, 1);
    250         int32_t inputStreams[1] =
    251                 { mZslStreamId };
    252         if (res == OK) request.update(ANDROID_REQUEST_INPUT_STREAMS,
    253                 inputStreams, 1);
    254         // TODO: Shouldn't we also update the latest preview frame?
    255         int32_t outputStreams[1] =
    256                 { client->getCaptureStreamId() };
    257         if (res == OK) request.update(ANDROID_REQUEST_OUTPUT_STREAMS,
    258                 outputStreams, 1);
    259         res = request.update(ANDROID_REQUEST_ID,
    260                 &requestId, 1);
    261 
    262         if (res != OK ) {
    263             ALOGE("%s: Unable to update frame to a reprocess request",
    264                   __FUNCTION__);
    265             return INVALID_OPERATION;
    266         }
    267 
    268         res = client->stopStream();
    269         if (res != OK) {
    270             ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: "
    271                 "%s (%d)",
    272                 __FUNCTION__, client->getCameraId(), strerror(-res), res);
    273             return INVALID_OPERATION;
    274         }
    275 
    276         // Update JPEG settings
    277         {
    278             SharedParameters::Lock l(client->getParameters());
    279             res = l.mParameters.updateRequestJpeg(&request);
    280             if (res != OK) {
    281                 ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL "
    282                         "capture request: %s (%d)", __FUNCTION__,
    283                         client->getCameraId(),
    284                         strerror(-res), res);
    285                 return res;
    286             }
    287         }
    288 
    289         mLatestCapturedRequest = request;
    290         res = client->getCameraDevice()->capture(request);
    291         if (res != OK ) {
    292             ALOGE("%s: Unable to send ZSL reprocess request to capture: %s"
    293                   " (%d)", __FUNCTION__, strerror(-res), res);
    294             return res;
    295         }
    296 
    297         mState = LOCKED;
    298     }
    299 
    300     return OK;
    301 }
    302 
    303 status_t ZslProcessor3::clearZslQueue() {
    304     Mutex::Autolock l(mInputMutex);
    305     // If in middle of capture, can't clear out queue
    306     if (mState == LOCKED) return OK;
    307 
    308     return clearZslQueueLocked();
    309 }
    310 
    311 status_t ZslProcessor3::clearZslQueueLocked() {
    312     if (mZslStream != 0) {
    313         return mZslStream->clearInputRingBuffer();
    314     }
    315     return OK;
    316 }
    317 
    318 void ZslProcessor3::dump(int fd, const Vector<String16>& /*args*/) const {
    319     Mutex::Autolock l(mInputMutex);
    320     if (!mLatestCapturedRequest.isEmpty()) {
    321         String8 result("    Latest ZSL capture request:\n");
    322         write(fd, result.string(), result.size());
    323         mLatestCapturedRequest.dump(fd, 2, 6);
    324     } else {
    325         String8 result("    Latest ZSL capture request: none yet\n");
    326         write(fd, result.string(), result.size());
    327     }
    328     dumpZslQueue(fd);
    329 }
    330 
    331 bool ZslProcessor3::threadLoop() {
    332     // TODO: remove dependency on thread. For now, shut thread down right
    333     // away.
    334     return false;
    335 }
    336 
    337 void ZslProcessor3::dumpZslQueue(int fd) const {
    338     String8 header("ZSL queue contents:");
    339     String8 indent("    ");
    340     ALOGV("%s", header.string());
    341     if (fd != -1) {
    342         header = indent + header + "\n";
    343         write(fd, header.string(), header.size());
    344     }
    345     for (size_t i = 0; i < mZslQueue.size(); i++) {
    346         const ZslPair &queueEntry = mZslQueue[i];
    347         nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
    348         camera_metadata_ro_entry_t entry;
    349         nsecs_t frameTimestamp = 0;
    350         int frameAeState = -1;
    351         if (!queueEntry.frame.isEmpty()) {
    352             entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
    353             if (entry.count > 0) frameTimestamp = entry.data.i64[0];
    354             entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE);
    355             if (entry.count > 0) frameAeState = entry.data.u8[0];
    356         }
    357         String8 result =
    358                 String8::format("   %d: b: %lld\tf: %lld, AE state: %d", i,
    359                         bufferTimestamp, frameTimestamp, frameAeState);
    360         ALOGV("%s", result.string());
    361         if (fd != -1) {
    362             result = indent + result + "\n";
    363             write(fd, result.string(), result.size());
    364         }
    365 
    366     }
    367 }
    368 
    369 nsecs_t ZslProcessor3::getCandidateTimestampLocked(size_t* metadataIdx) const {
    370     /**
    371      * Find the smallest timestamp we know about so far
    372      * - ensure that aeState is either converged or locked
    373      */
    374 
    375     size_t idx = 0;
    376     nsecs_t minTimestamp = -1;
    377 
    378     size_t emptyCount = mFrameList.size();
    379 
    380     for (size_t j = 0; j < mFrameList.size(); j++) {
    381         const CameraMetadata &frame = mFrameList[j];
    382         if (!frame.isEmpty()) {
    383 
    384             emptyCount--;
    385 
    386             camera_metadata_ro_entry_t entry;
    387             entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
    388             if (entry.count == 0) {
    389                 ALOGE("%s: Can't find timestamp in frame!",
    390                         __FUNCTION__);
    391                 continue;
    392             }
    393             nsecs_t frameTimestamp = entry.data.i64[0];
    394             if (minTimestamp > frameTimestamp || minTimestamp == -1) {
    395 
    396                 entry = frame.find(ANDROID_CONTROL_AE_STATE);
    397 
    398                 if (entry.count == 0) {
    399                     /**
    400                      * This is most likely a HAL bug. The aeState field is
    401                      * mandatory, so it should always be in a metadata packet.
    402                      */
    403                     ALOGW("%s: ZSL queue frame has no AE state field!",
    404                             __FUNCTION__);
    405                     continue;
    406                 }
    407                 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED &&
    408                         entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) {
    409                     ALOGVV("%s: ZSL queue frame AE state is %d, need "
    410                            "full capture",  __FUNCTION__, entry.data.u8[0]);
    411                     continue;
    412                 }
    413 
    414                 minTimestamp = frameTimestamp;
    415                 idx = j;
    416             }
    417 
    418             ALOGVV("%s: Saw timestamp %lld", __FUNCTION__, frameTimestamp);
    419         }
    420     }
    421 
    422     if (emptyCount == mFrameList.size()) {
    423         /**
    424          * This could be mildly bad and means our ZSL was triggered before
    425          * there were any frames yet received by the camera framework.
    426          *
    427          * This is a fairly corner case which can happen under:
    428          * + a user presses the shutter button real fast when the camera starts
    429          *     (startPreview followed immediately by takePicture).
    430          * + burst capture case (hitting shutter button as fast possible)
    431          *
    432          * If this happens in steady case (preview running for a while, call
    433          *     a single takePicture) then this might be a fwk bug.
    434          */
    435         ALOGW("%s: ZSL queue has no metadata frames", __FUNCTION__);
    436     }
    437 
    438     ALOGV("%s: Candidate timestamp %lld (idx %d), empty frames: %d",
    439           __FUNCTION__, minTimestamp, idx, emptyCount);
    440 
    441     if (metadataIdx) {
    442         *metadataIdx = idx;
    443     }
    444 
    445     return minTimestamp;
    446 }
    447 
    448 void ZslProcessor3::onBufferAcquired(const BufferInfo& /*bufferInfo*/) {
    449     // Intentionally left empty
    450     // Although theoretically we could use this to get better dump info
    451 }
    452 
    453 void ZslProcessor3::onBufferReleased(const BufferInfo& bufferInfo) {
    454     Mutex::Autolock l(mInputMutex);
    455 
    456     // ignore output buffers
    457     if (bufferInfo.mOutput) {
    458         return;
    459     }
    460 
    461     // TODO: Verify that the buffer is in our queue by looking at timestamp
    462     // theoretically unnecessary unless we change the following assumptions:
    463     // -- only 1 buffer reprocessed at a time (which is the case now)
    464 
    465     // Erase entire ZSL queue since we've now completed the capture and preview
    466     // is stopped.
    467     //
    468     // We need to guarantee that if we do two back-to-back captures,
    469     // the second won't use a buffer that's older/the same as the first, which
    470     // is theoretically possible if we don't clear out the queue and the
    471     // selection criteria is something like 'newest'. Clearing out the queue
    472     // on a completed capture ensures we'll only use new data.
    473     ALOGV("%s: Memory optimization, clearing ZSL queue",
    474           __FUNCTION__);
    475     clearZslQueueLocked();
    476 
    477     // Required so we accept more ZSL requests
    478     mState = RUNNING;
    479 }
    480 
    481 }; // namespace camera2
    482 }; // namespace android
    483