Home | History | Annotate | Download | only in api2 
      1 /*
      2  * Copyright (C) 2019 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #define LOG_TAG "Camera3-HeicCompositeStream"
     18 #define ATRACE_TAG ATRACE_TAG_CAMERA
     19 //#define LOG_NDEBUG 0
     20 
     21 #include <linux/memfd.h>
     22 #include <pthread.h>
     23 #include <sys/syscall.h>
     24 
     25 #include <android/hardware/camera/device/3.5/types.h>
     26 #include <libyuv.h>
     27 #include <gui/Surface.h>
     28 #include <utils/Log.h>
     29 #include <utils/Trace.h>
     30 
     31 #include <media/ICrypto.h>
     32 #include <media/MediaCodecBuffer.h>
     33 #include <media/stagefright/foundation/ABuffer.h>
     34 #include <media/stagefright/foundation/AMessage.h>
     35 #include <media/stagefright/foundation/MediaDefs.h>
     36 #include <media/stagefright/MediaCodecConstants.h>
     37 
     38 #include "common/CameraDeviceBase.h"
     39 #include "utils/ExifUtils.h"
     40 #include "HeicEncoderInfoManager.h"
     41 #include "HeicCompositeStream.h"
     42 
     43 using android::hardware::camera::device::V3_5::CameraBlob;
     44 using android::hardware::camera::device::V3_5::CameraBlobId;
     45 
     46 namespace android {
     47 namespace camera3 {
     48 
     49 HeicCompositeStream::HeicCompositeStream(wp<CameraDeviceBase> device,
     50         wp<hardware::camera2::ICameraDeviceCallbacks> cb) :
     51         CompositeStream(device, cb),
     52         mUseHeic(false),
     53         mNumOutputTiles(1),
     54         mOutputWidth(0),
     55         mOutputHeight(0),
     56         mMaxHeicBufferSize(0),
     57         mGridWidth(HeicEncoderInfoManager::kGridWidth),
     58         mGridHeight(HeicEncoderInfoManager::kGridHeight),
     59         mGridRows(1),
     60         mGridCols(1),
     61         mUseGrid(false),
     62         mAppSegmentStreamId(-1),
     63         mAppSegmentSurfaceId(-1),
     64         mAppSegmentBufferAcquired(false),
     65         mMainImageStreamId(-1),
     66         mMainImageSurfaceId(-1),
     67         mYuvBufferAcquired(false),
     68         mProducerListener(new ProducerListener()),
     69         mOutputBufferCounter(0),
     70         mGridTimestampUs(0) {
     71 }
     72 
     73 HeicCompositeStream::~HeicCompositeStream() {
     74     // Call deinitCodec in case stream hasn't been deleted yet to avoid any
     75     // memory/resource leak.
     76     deinitCodec();
     77 
     78     mInputAppSegmentBuffers.clear();
     79     mCodecOutputBuffers.clear();
     80 
     81     mAppSegmentStreamId = -1;
     82     mAppSegmentSurfaceId = -1;
     83     mAppSegmentConsumer.clear();
     84     mAppSegmentSurface.clear();
     85 
     86     mMainImageStreamId = -1;
     87     mMainImageSurfaceId = -1;
     88     mMainImageConsumer.clear();
     89     mMainImageSurface.clear();
     90 }
     91 
     92 bool HeicCompositeStream::isHeicCompositeStream(const sp<Surface> &surface) {
     93     ANativeWindow *anw = surface.get();
     94     status_t err;
     95     int format;
     96     if ((err = anw->query(anw, NATIVE_WINDOW_FORMAT, &format)) != OK) {
     97         String8 msg = String8::format("Failed to query Surface format: %s (%d)", strerror(-err),
     98                 err);
     99         ALOGE("%s: %s", __FUNCTION__, msg.string());
    100         return false;
    101     }
    102 
    103     int dataspace;
    104     if ((err = anw->query(anw, NATIVE_WINDOW_DEFAULT_DATASPACE, &dataspace)) != OK) {
    105         String8 msg = String8::format("Failed to query Surface dataspace: %s (%d)", strerror(-err),
    106                 err);
    107         ALOGE("%s: %s", __FUNCTION__, msg.string());
    108         return false;
    109     }
    110 
    111     return ((format == HAL_PIXEL_FORMAT_BLOB) && (dataspace == HAL_DATASPACE_HEIF));
    112 }
    113 
    114 status_t HeicCompositeStream::createInternalStreams(const std::vector<sp<Surface>>& consumers,
    115         bool /*hasDeferredConsumer*/, uint32_t width, uint32_t height, int format,
    116         camera3_stream_rotation_t rotation, int *id, const String8& physicalCameraId,
    117         std::vector<int> *surfaceIds, int /*streamSetId*/, bool /*isShared*/) {
    118 
    119     sp<CameraDeviceBase> device = mDevice.promote();
    120     if (!device.get()) {
    121         ALOGE("%s: Invalid camera device!", __FUNCTION__);
    122         return NO_INIT;
    123     }
    124 
    125     status_t res = initializeCodec(width, height, device);
    126     if (res != OK) {
    127         ALOGE("%s: Failed to initialize HEIC/HEVC codec: %s (%d)",
    128                 __FUNCTION__, strerror(-res), res);
    129         return NO_INIT;
    130     }
    131 
    132     sp<IGraphicBufferProducer> producer;
    133     sp<IGraphicBufferConsumer> consumer;
    134     BufferQueue::createBufferQueue(&producer, &consumer);
    135     mAppSegmentConsumer = new CpuConsumer(consumer, 1);
    136     mAppSegmentConsumer->setFrameAvailableListener(this);
    137     mAppSegmentConsumer->setName(String8("Camera3-HeicComposite-AppSegmentStream"));
    138     mAppSegmentSurface = new Surface(producer);
    139 
    140     mStaticInfo = device->info();
    141 
    142     res = device->createStream(mAppSegmentSurface, mAppSegmentMaxSize, 1, format,
    143             kAppSegmentDataSpace, rotation, &mAppSegmentStreamId, physicalCameraId, surfaceIds);
    144     if (res == OK) {
    145         mAppSegmentSurfaceId = (*surfaceIds)[0];
    146     } else {
    147         ALOGE("%s: Failed to create JPEG App segment stream: %s (%d)", __FUNCTION__,
    148                 strerror(-res), res);
    149         return res;
    150     }
    151 
    152     if (!mUseGrid) {
    153         res = mCodec->createInputSurface(&producer);
    154         if (res != OK) {
    155             ALOGE("%s: Failed to create input surface for Heic codec: %s (%d)",
    156                     __FUNCTION__, strerror(-res), res);
    157             return res;
    158         }
    159     } else {
    160         BufferQueue::createBufferQueue(&producer, &consumer);
    161         mMainImageConsumer = new CpuConsumer(consumer, 1);
    162         mMainImageConsumer->setFrameAvailableListener(this);
    163         mMainImageConsumer->setName(String8("Camera3-HeicComposite-HevcInputYUVStream"));
    164     }
    165     mMainImageSurface = new Surface(producer);
    166 
    167     res = mCodec->start();
    168     if (res != OK) {
    169         ALOGE("%s: Failed to start codec: %s (%d)", __FUNCTION__,
    170                 strerror(-res), res);
    171         return res;
    172     }
    173 
    174     std::vector<int> sourceSurfaceId;
    175     //Use YUV_888 format if framework tiling is needed.
    176     int srcStreamFmt = mUseGrid ? HAL_PIXEL_FORMAT_YCbCr_420_888 :
    177             HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
    178     res = device->createStream(mMainImageSurface, width, height, srcStreamFmt, kHeifDataSpace,
    179             rotation, id, physicalCameraId, &sourceSurfaceId);
    180     if (res == OK) {
    181         mMainImageSurfaceId = sourceSurfaceId[0];
    182         mMainImageStreamId = *id;
    183     } else {
    184         ALOGE("%s: Failed to create main image stream: %s (%d)", __FUNCTION__,
    185                 strerror(-res), res);
    186         return res;
    187     }
    188 
    189     mOutputSurface = consumers[0];
    190     res = registerCompositeStreamListener(getStreamId());
    191     if (res != OK) {
    192         ALOGE("%s: Failed to register HAL main image stream", __FUNCTION__);
    193         return res;
    194     }
    195 
    196     initCopyRowFunction(width);
    197     return res;
    198 }
    199 
    200 status_t HeicCompositeStream::deleteInternalStreams() {
    201     requestExit();
    202     auto res = join();
    203     if (res != OK) {
    204         ALOGE("%s: Failed to join with the main processing thread: %s (%d)", __FUNCTION__,
    205                 strerror(-res), res);
    206     }
    207 
    208     deinitCodec();
    209 
    210     if (mAppSegmentStreamId >= 0) {
    211         sp<CameraDeviceBase> device = mDevice.promote();
    212         if (!device.get()) {
    213             ALOGE("%s: Invalid camera device!", __FUNCTION__);
    214             return NO_INIT;
    215         }
    216 
    217         res = device->deleteStream(mAppSegmentStreamId);
    218         mAppSegmentStreamId = -1;
    219     }
    220 
    221     if (mOutputSurface != nullptr) {
    222         mOutputSurface->disconnect(NATIVE_WINDOW_API_CAMERA);
    223         mOutputSurface.clear();
    224     }
    225     return res;
    226 }
    227 
    228 void HeicCompositeStream::onBufferReleased(const BufferInfo& bufferInfo) {
    229     Mutex::Autolock l(mMutex);
    230 
    231     if (bufferInfo.mError) return;
    232 
    233     mCodecOutputBufferTimestamps.push(bufferInfo.mTimestamp);
    234 }
    235 
    236 // We need to get the settings early to handle the case where the codec output
    237 // arrives earlier than result metadata.
    238 void HeicCompositeStream::onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId,
    239         const CameraMetadata& settings) {
    240     ATRACE_ASYNC_BEGIN("HEIC capture", frameNumber);
    241 
    242     Mutex::Autolock l(mMutex);
    243     if (mErrorState || (streamId != getStreamId())) {
    244         return;
    245     }
    246 
    247     mPendingCaptureResults.emplace(frameNumber, CameraMetadata());
    248 
    249     camera_metadata_ro_entry entry;
    250 
    251     int32_t orientation = 0;
    252     entry = settings.find(ANDROID_JPEG_ORIENTATION);
    253     if (entry.count == 1) {
    254         orientation = entry.data.i32[0];
    255     }
    256 
    257     int32_t quality = kDefaultJpegQuality;
    258     entry = settings.find(ANDROID_JPEG_QUALITY);
    259     if (entry.count == 1) {
    260         quality = entry.data.i32[0];
    261     }
    262 
    263     mSettingsByFrameNumber[frameNumber] = std::make_pair(orientation, quality);
    264 }
    265 
    266 void HeicCompositeStream::onFrameAvailable(const BufferItem& item) {
    267     if (item.mDataSpace == static_cast<android_dataspace>(kAppSegmentDataSpace)) {
    268         ALOGV("%s: JPEG APP segments buffer with ts: %" PRIu64 " ms. arrived!",
    269                 __func__, ns2ms(item.mTimestamp));
    270 
    271         Mutex::Autolock l(mMutex);
    272         if (!mErrorState) {
    273             mInputAppSegmentBuffers.push_back(item.mTimestamp);
    274             mInputReadyCondition.signal();
    275         }
    276     } else if (item.mDataSpace == kHeifDataSpace) {
    277         ALOGV("%s: YUV_888 buffer with ts: %" PRIu64 " ms. arrived!",
    278                 __func__, ns2ms(item.mTimestamp));
    279 
    280         Mutex::Autolock l(mMutex);
    281         if (!mUseGrid) {
    282             ALOGE("%s: YUV_888 internal stream is only supported for HEVC tiling",
    283                     __FUNCTION__);
    284             return;
    285         }
    286         if (!mErrorState) {
    287             mInputYuvBuffers.push_back(item.mTimestamp);
    288             mInputReadyCondition.signal();
    289         }
    290     } else {
    291         ALOGE("%s: Unexpected data space: 0x%x", __FUNCTION__, item.mDataSpace);
    292     }
    293 }
    294 
    295 status_t HeicCompositeStream::getCompositeStreamInfo(const OutputStreamInfo &streamInfo,
    296             const CameraMetadata& ch, std::vector<OutputStreamInfo>* compositeOutput /*out*/) {
    297     if (compositeOutput == nullptr) {
    298         return BAD_VALUE;
    299     }
    300 
    301     compositeOutput->clear();
    302 
    303     bool useGrid, useHeic;
    304     bool isSizeSupported = isSizeSupportedByHeifEncoder(
    305             streamInfo.width, streamInfo.height, &useHeic, &useGrid, nullptr);
    306     if (!isSizeSupported) {
    307         // Size is not supported by either encoder.
    308         return OK;
    309     }
    310 
    311     compositeOutput->insert(compositeOutput->end(), 2, streamInfo);
    312 
    313     // JPEG APPS segments Blob stream info
    314     (*compositeOutput)[0].width = calcAppSegmentMaxSize(ch);
    315     (*compositeOutput)[0].height = 1;
    316     (*compositeOutput)[0].format = HAL_PIXEL_FORMAT_BLOB;
    317     (*compositeOutput)[0].dataSpace = kAppSegmentDataSpace;
    318     (*compositeOutput)[0].consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN;
    319 
    320     // YUV/IMPLEMENTATION_DEFINED stream info
    321     (*compositeOutput)[1].width = streamInfo.width;
    322     (*compositeOutput)[1].height = streamInfo.height;
    323     (*compositeOutput)[1].format = useGrid ? HAL_PIXEL_FORMAT_YCbCr_420_888 :
    324             HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
    325     (*compositeOutput)[1].dataSpace = kHeifDataSpace;
    326     (*compositeOutput)[1].consumerUsage = useHeic ? GRALLOC_USAGE_HW_IMAGE_ENCODER :
    327             useGrid ? GRALLOC_USAGE_SW_READ_OFTEN : GRALLOC_USAGE_HW_VIDEO_ENCODER;
    328 
    329     return NO_ERROR;
    330 }
    331 
    332 bool HeicCompositeStream::isSizeSupportedByHeifEncoder(int32_t width, int32_t height,
    333         bool* useHeic, bool* useGrid, int64_t* stall, AString* hevcName) {
    334     static HeicEncoderInfoManager& heicManager = HeicEncoderInfoManager::getInstance();
    335     return heicManager.isSizeSupported(width, height, useHeic, useGrid, stall, hevcName);
    336 }
    337 
    338 bool HeicCompositeStream::isInMemoryTempFileSupported() {
    339     int memfd = syscall(__NR_memfd_create, "HEIF-try-memfd", MFD_CLOEXEC);
    340     if (memfd == -1) {
    341         if (errno != ENOSYS) {
    342             ALOGE("%s: Failed to create tmpfs file. errno %d", __FUNCTION__, errno);
    343         }
    344         return false;
    345     }
    346     close(memfd);
    347     return true;
    348 }
    349 
    350 void HeicCompositeStream::onHeicOutputFrameAvailable(
    351         const CodecOutputBufferInfo& outputBufferInfo) {
    352     Mutex::Autolock l(mMutex);
    353 
    354     ALOGV("%s: index %d, offset %d, size %d, time %" PRId64 ", flags 0x%x",
    355             __FUNCTION__, outputBufferInfo.index, outputBufferInfo.offset,
    356             outputBufferInfo.size, outputBufferInfo.timeUs, outputBufferInfo.flags);
    357 
    358     if (!mErrorState) {
    359         if ((outputBufferInfo.size > 0) &&
    360                 ((outputBufferInfo.flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) == 0)) {
    361             mCodecOutputBuffers.push_back(outputBufferInfo);
    362             mInputReadyCondition.signal();
    363         } else {
    364             mCodec->releaseOutputBuffer(outputBufferInfo.index);
    365         }
    366     } else {
    367         mCodec->releaseOutputBuffer(outputBufferInfo.index);
    368     }
    369 }
    370 
    371 void HeicCompositeStream::onHeicInputFrameAvailable(int32_t index) {
    372     Mutex::Autolock l(mMutex);
    373 
    374     if (!mUseGrid) {
    375         ALOGE("%s: Codec YUV input mode must only be used for Hevc tiling mode", __FUNCTION__);
    376         return;
    377     }
    378 
    379     mCodecInputBuffers.push_back(index);
    380     mInputReadyCondition.signal();
    381 }
    382 
    383 void HeicCompositeStream::onHeicFormatChanged(sp<AMessage>& newFormat) {
    384     if (newFormat == nullptr) {
    385         ALOGE("%s: newFormat must not be null!", __FUNCTION__);
    386         return;
    387     }
    388 
    389     Mutex::Autolock l(mMutex);
    390 
    391     AString mime;
    392     AString mimeHeic(MIMETYPE_IMAGE_ANDROID_HEIC);
    393     newFormat->findString(KEY_MIME, &mime);
    394     if (mime != mimeHeic) {
    395         // For HEVC codec, below keys need to be filled out or overwritten so that the
    396         // muxer can handle them as HEIC output image.
    397         newFormat->setString(KEY_MIME, mimeHeic);
    398         newFormat->setInt32(KEY_WIDTH, mOutputWidth);
    399         newFormat->setInt32(KEY_HEIGHT, mOutputHeight);
    400         if (mUseGrid) {
    401             newFormat->setInt32(KEY_TILE_WIDTH, mGridWidth);
    402             newFormat->setInt32(KEY_TILE_HEIGHT, mGridHeight);
    403             newFormat->setInt32(KEY_GRID_ROWS, mGridRows);
    404             newFormat->setInt32(KEY_GRID_COLUMNS, mGridCols);
    405         }
    406     }
    407     newFormat->setInt32(KEY_IS_DEFAULT, 1 /*isPrimary*/);
    408 
    409     int32_t gridRows, gridCols;
    410     if (newFormat->findInt32(KEY_GRID_ROWS, &gridRows) &&
    411             newFormat->findInt32(KEY_GRID_COLUMNS, &gridCols)) {
    412         mNumOutputTiles = gridRows * gridCols;
    413     } else {
    414         mNumOutputTiles = 1;
    415     }
    416 
    417     ALOGV("%s: mNumOutputTiles is %zu", __FUNCTION__, mNumOutputTiles);
    418     mFormat = newFormat;
    419 }
    420 
    421 void HeicCompositeStream::onHeicCodecError() {
    422     Mutex::Autolock l(mMutex);
    423     mErrorState = true;
    424 }
    425 
    426 status_t HeicCompositeStream::configureStream() {
    427     if (isRunning()) {
    428         // Processing thread is already running, nothing more to do.
    429         return NO_ERROR;
    430     }
    431 
    432     if (mOutputSurface.get() == nullptr) {
    433         ALOGE("%s: No valid output surface set!", __FUNCTION__);
    434         return NO_INIT;
    435     }
    436 
    437     auto res = mOutputSurface->connect(NATIVE_WINDOW_API_CAMERA, mProducerListener);
    438     if (res != OK) {
    439         ALOGE("%s: Unable to connect to native window for stream %d",
    440                 __FUNCTION__, mMainImageStreamId);
    441         return res;
    442     }
    443 
    444     if ((res = native_window_set_buffers_format(mOutputSurface.get(), HAL_PIXEL_FORMAT_BLOB))
    445             != OK) {
    446         ALOGE("%s: Unable to configure stream buffer format for stream %d", __FUNCTION__,
    447                 mMainImageStreamId);
    448         return res;
    449     }
    450 
    451     ANativeWindow *anwConsumer = mOutputSurface.get();
    452     int maxConsumerBuffers;
    453     if ((res = anwConsumer->query(anwConsumer, NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS,
    454                     &maxConsumerBuffers)) != OK) {
    455         ALOGE("%s: Unable to query consumer undequeued"
    456                 " buffer count for stream %d", __FUNCTION__, mMainImageStreamId);
    457         return res;
    458     }
    459 
    460     // Cannot use SourceSurface buffer count since it could be codec's 512*512 tile
    461     // buffer count.
    462     int maxProducerBuffers = 1;
    463     if ((res = native_window_set_buffer_count(
    464                     anwConsumer, maxProducerBuffers + maxConsumerBuffers)) != OK) {
    465         ALOGE("%s: Unable to set buffer count for stream %d", __FUNCTION__, mMainImageStreamId);
    466         return res;
    467     }
    468 
    469     if ((res = native_window_set_buffers_dimensions(anwConsumer, mMaxHeicBufferSize, 1)) != OK) {
    470         ALOGE("%s: Unable to set buffer dimension %zu x 1 for stream %d: %s (%d)",
    471                 __FUNCTION__, mMaxHeicBufferSize, mMainImageStreamId, strerror(-res), res);
    472         return res;
    473     }
    474 
    475     run("HeicCompositeStreamProc");
    476 
    477     return NO_ERROR;
    478 }
    479 
    480 status_t HeicCompositeStream::insertGbp(SurfaceMap* /*out*/outSurfaceMap,
    481         Vector<int32_t>* /*out*/outputStreamIds, int32_t* /*out*/currentStreamId) {
    482     if (outSurfaceMap->find(mAppSegmentStreamId) == outSurfaceMap->end()) {
    483         (*outSurfaceMap)[mAppSegmentStreamId] = std::vector<size_t>();
    484         outputStreamIds->push_back(mAppSegmentStreamId);
    485     }
    486     (*outSurfaceMap)[mAppSegmentStreamId].push_back(mAppSegmentSurfaceId);
    487 
    488     if (outSurfaceMap->find(mMainImageStreamId) == outSurfaceMap->end()) {
    489         (*outSurfaceMap)[mMainImageStreamId] = std::vector<size_t>();
    490         outputStreamIds->push_back(mMainImageStreamId);
    491     }
    492     (*outSurfaceMap)[mMainImageStreamId].push_back(mMainImageSurfaceId);
    493 
    494     if (currentStreamId != nullptr) {
    495         *currentStreamId = mMainImageStreamId;
    496     }
    497 
    498     return NO_ERROR;
    499 }
    500 
    501 void HeicCompositeStream::onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) {
    502     Mutex::Autolock l(mMutex);
    503     if (mErrorState) {
    504         return;
    505     }
    506 
    507     if (mSettingsByFrameNumber.find(resultExtras.frameNumber) != mSettingsByFrameNumber.end()) {
    508         mFrameNumberMap.emplace(resultExtras.frameNumber, timestamp);
    509         mSettingsByTimestamp[timestamp] = mSettingsByFrameNumber[resultExtras.frameNumber];
    510         mSettingsByFrameNumber.erase(resultExtras.frameNumber);
    511         mInputReadyCondition.signal();
    512     }
    513 }
    514 
    515 void HeicCompositeStream::compilePendingInputLocked() {
    516     while (!mSettingsByTimestamp.empty()) {
    517         auto it = mSettingsByTimestamp.begin();
    518         mPendingInputFrames[it->first].orientation = it->second.first;
    519         mPendingInputFrames[it->first].quality = it->second.second;
    520         mSettingsByTimestamp.erase(it);
    521     }
    522 
    523     while (!mInputAppSegmentBuffers.empty() && !mAppSegmentBufferAcquired) {
    524         CpuConsumer::LockedBuffer imgBuffer;
    525         auto it = mInputAppSegmentBuffers.begin();
    526         auto res = mAppSegmentConsumer->lockNextBuffer(&imgBuffer);
    527         if (res == NOT_ENOUGH_DATA) {
    528             // Canot not lock any more buffers.
    529             break;
    530         } else if ((res != OK) || (*it != imgBuffer.timestamp)) {
    531             if (res != OK) {
    532                 ALOGE("%s: Error locking JPEG_APP_SEGMENTS image buffer: %s (%d)", __FUNCTION__,
    533                         strerror(-res), res);
    534             } else {
    535                 ALOGE("%s: Expecting JPEG_APP_SEGMENTS buffer with time stamp: %" PRId64
    536                         " received buffer with time stamp: %" PRId64, __FUNCTION__,
    537                         *it, imgBuffer.timestamp);
    538             }
    539             mPendingInputFrames[*it].error = true;
    540             mInputAppSegmentBuffers.erase(it);
    541             continue;
    542         }
    543 
    544         if ((mPendingInputFrames.find(imgBuffer.timestamp) != mPendingInputFrames.end()) &&
    545                 (mPendingInputFrames[imgBuffer.timestamp].error)) {
    546             mAppSegmentConsumer->unlockBuffer(imgBuffer);
    547         } else {
    548             mPendingInputFrames[imgBuffer.timestamp].appSegmentBuffer = imgBuffer;
    549             mAppSegmentBufferAcquired = true;
    550         }
    551         mInputAppSegmentBuffers.erase(it);
    552     }
    553 
    554     while (!mInputYuvBuffers.empty() && !mYuvBufferAcquired) {
    555         CpuConsumer::LockedBuffer imgBuffer;
    556         auto it = mInputYuvBuffers.begin();
    557         auto res = mMainImageConsumer->lockNextBuffer(&imgBuffer);
    558         if (res == NOT_ENOUGH_DATA) {
    559             // Canot not lock any more buffers.
    560             break;
    561         } else if (res != OK) {
    562             ALOGE("%s: Error locking YUV_888 image buffer: %s (%d)", __FUNCTION__,
    563                     strerror(-res), res);
    564             mPendingInputFrames[*it].error = true;
    565             mInputYuvBuffers.erase(it);
    566             continue;
    567         } else if (*it != imgBuffer.timestamp) {
    568             ALOGW("%s: Expecting YUV_888 buffer with time stamp: %" PRId64 " received buffer with "
    569                     "time stamp: %" PRId64, __FUNCTION__, *it, imgBuffer.timestamp);
    570             mPendingInputFrames[*it].error = true;
    571             mInputYuvBuffers.erase(it);
    572             continue;
    573         }
    574 
    575         if ((mPendingInputFrames.find(imgBuffer.timestamp) != mPendingInputFrames.end()) &&
    576                 (mPendingInputFrames[imgBuffer.timestamp].error)) {
    577             mMainImageConsumer->unlockBuffer(imgBuffer);
    578         } else {
    579             mPendingInputFrames[imgBuffer.timestamp].yuvBuffer = imgBuffer;
    580             mYuvBufferAcquired = true;
    581         }
    582         mInputYuvBuffers.erase(it);
    583     }
    584 
    585     while (!mCodecOutputBuffers.empty()) {
    586         auto it = mCodecOutputBuffers.begin();
    587         // Bitstream buffer timestamp doesn't necessarily directly correlate with input
    588         // buffer timestamp. Assume encoder input to output is FIFO, use a queue
    589         // to look up timestamp.
    590         int64_t bufferTime = -1;
    591         if (mCodecOutputBufferTimestamps.empty()) {
    592             ALOGE("%s: Failed to find buffer timestamp for codec output buffer!", __FUNCTION__);
    593         } else {
    594             // Direct mapping between camera timestamp (in ns) and codec timestamp (in us).
    595             bufferTime = mCodecOutputBufferTimestamps.front();
    596             mOutputBufferCounter++;
    597             if (mOutputBufferCounter == mNumOutputTiles) {
    598                 mCodecOutputBufferTimestamps.pop();
    599                 mOutputBufferCounter = 0;
    600             }
    601 
    602             mPendingInputFrames[bufferTime].codecOutputBuffers.push_back(*it);
    603         }
    604         mCodecOutputBuffers.erase(it);
    605     }
    606 
    607     while (!mFrameNumberMap.empty()) {
    608         auto it = mFrameNumberMap.begin();
    609         mPendingInputFrames[it->second].frameNumber = it->first;
    610         mFrameNumberMap.erase(it);
    611     }
    612 
    613     while (!mCaptureResults.empty()) {
    614         auto it = mCaptureResults.begin();
    615         // Negative timestamp indicates that something went wrong during the capture result
    616         // collection process.
    617         if (it->first >= 0) {
    618             if (mPendingInputFrames[it->first].frameNumber == std::get<0>(it->second)) {
    619                 mPendingInputFrames[it->first].result =
    620                         std::make_unique<CameraMetadata>(std::get<1>(it->second));
    621             } else {
    622                 ALOGE("%s: Capture result frameNumber/timestamp mapping changed between "
    623                         "shutter and capture result!", __FUNCTION__);
    624             }
    625         }
    626         mCaptureResults.erase(it);
    627     }
    628 
    629     // mErrorFrameNumbers stores frame number of dropped buffers.
    630     auto it = mErrorFrameNumbers.begin();
    631     while (it != mErrorFrameNumbers.end()) {
    632         bool frameFound = false;
    633         for (auto &inputFrame : mPendingInputFrames) {
    634             if (inputFrame.second.frameNumber == *it) {
    635                 inputFrame.second.error = true;
    636                 frameFound = true;
    637                 break;
    638             }
    639         }
    640 
    641         if (frameFound) {
    642             it = mErrorFrameNumbers.erase(it);
    643         } else {
    644             ALOGW("%s: Not able to find failing input with frame number: %" PRId64, __FUNCTION__,
    645                     *it);
    646             it++;
    647         }
    648     }
    649 
    650     // Distribute codec input buffers to be filled out from YUV output
    651     for (auto it = mPendingInputFrames.begin();
    652             it != mPendingInputFrames.end() && mCodecInputBuffers.size() > 0; it++) {
    653         InputFrame& inputFrame(it->second);
    654         if (inputFrame.codecInputCounter < mGridRows * mGridCols) {
    655             // Available input tiles that are required for the current input
    656             // image.
    657             size_t newInputTiles = std::min(mCodecInputBuffers.size(),
    658                     mGridRows * mGridCols - inputFrame.codecInputCounter);
    659             for (size_t i = 0; i < newInputTiles; i++) {
    660                 CodecInputBufferInfo inputInfo =
    661                         { mCodecInputBuffers[0], mGridTimestampUs++, inputFrame.codecInputCounter };
    662                 inputFrame.codecInputBuffers.push_back(inputInfo);
    663 
    664                 mCodecInputBuffers.erase(mCodecInputBuffers.begin());
    665                 inputFrame.codecInputCounter++;
    666             }
    667             break;
    668         }
    669     }
    670 }
    671 
    672 bool HeicCompositeStream::getNextReadyInputLocked(int64_t *currentTs /*out*/) {
    673     if (currentTs == nullptr) {
    674         return false;
    675     }
    676 
    677     bool newInputAvailable = false;
    678     for (const auto& it : mPendingInputFrames) {
    679         bool appSegmentReady = (it.second.appSegmentBuffer.data != nullptr) &&
    680                 !it.second.appSegmentWritten && it.second.result != nullptr;
    681         bool codecOutputReady = !it.second.codecOutputBuffers.empty();
    682         bool codecInputReady = (it.second.yuvBuffer.data != nullptr) &&
    683                 (!it.second.codecInputBuffers.empty());
    684         if ((!it.second.error) &&
    685                 (it.first < *currentTs) &&
    686                 (appSegmentReady || codecOutputReady || codecInputReady)) {
    687             *currentTs = it.first;
    688             newInputAvailable = true;
    689             break;
    690         }
    691     }
    692 
    693     return newInputAvailable;
    694 }
    695 
    696 int64_t HeicCompositeStream::getNextFailingInputLocked(int64_t *currentTs /*out*/) {
    697     int64_t res = -1;
    698     if (currentTs == nullptr) {
    699         return res;
    700     }
    701 
    702     for (const auto& it : mPendingInputFrames) {
    703         if (it.second.error && !it.second.errorNotified && (it.first < *currentTs)) {
    704             *currentTs = it.first;
    705             res = it.second.frameNumber;
    706             break;
    707         }
    708     }
    709 
    710     return res;
    711 }
    712 
    713 status_t HeicCompositeStream::processInputFrame(nsecs_t timestamp,
    714         InputFrame &inputFrame) {
    715     ATRACE_CALL();
    716     status_t res = OK;
    717 
    718     bool appSegmentReady = inputFrame.appSegmentBuffer.data != nullptr &&
    719             !inputFrame.appSegmentWritten && inputFrame.result != nullptr;
    720     bool codecOutputReady = inputFrame.codecOutputBuffers.size() > 0;
    721     bool codecInputReady = inputFrame.yuvBuffer.data != nullptr &&
    722            !inputFrame.codecInputBuffers.empty();
    723 
    724     if (!appSegmentReady && !codecOutputReady && !codecInputReady) {
    725         ALOGW("%s: No valid appSegmentBuffer/codec input/outputBuffer available!", __FUNCTION__);
    726         return OK;
    727     }
    728 
    729     // Handle inputs for Hevc tiling
    730     if (codecInputReady) {
    731         res = processCodecInputFrame(inputFrame);
    732         if (res != OK) {
    733             ALOGE("%s: Failed to process codec input frame: %s (%d)", __FUNCTION__,
    734                     strerror(-res), res);
    735             return res;
    736         }
    737     }
    738 
    739     // Initialize and start muxer if not yet done so
    740     if (inputFrame.muxer == nullptr) {
    741         res = startMuxerForInputFrame(timestamp, inputFrame);
    742         if (res != OK) {
    743             ALOGE("%s: Failed to create and start muxer: %s (%d)", __FUNCTION__,
    744                     strerror(-res), res);
    745             return res;
    746         }
    747     }
    748 
    749     // Write JPEG APP segments data to the muxer.
    750     if (appSegmentReady && inputFrame.muxer != nullptr) {
    751         res = processAppSegment(timestamp, inputFrame);
    752         if (res != OK) {
    753             ALOGE("%s: Failed to process JPEG APP segments: %s (%d)", __FUNCTION__,
    754                     strerror(-res), res);
    755             return res;
    756         }
    757     }
    758 
    759     // Write media codec bitstream buffers to muxer.
    760     while (!inputFrame.codecOutputBuffers.empty()) {
    761         res = processOneCodecOutputFrame(timestamp, inputFrame);
    762         if (res != OK) {
    763             ALOGE("%s: Failed to process codec output frame: %s (%d)", __FUNCTION__,
    764                     strerror(-res), res);
    765             return res;
    766         }
    767     }
    768 
    769     if (inputFrame.appSegmentWritten && inputFrame.pendingOutputTiles == 0) {
    770         res = processCompletedInputFrame(timestamp, inputFrame);
    771         if (res != OK) {
    772             ALOGE("%s: Failed to process completed input frame: %s (%d)", __FUNCTION__,
    773                     strerror(-res), res);
    774             return res;
    775         }
    776     }
    777 
    778     return res;
    779 }
    780 
    781 status_t HeicCompositeStream::startMuxerForInputFrame(nsecs_t timestamp, InputFrame &inputFrame) {
    782     sp<ANativeWindow> outputANW = mOutputSurface;
    783     if (inputFrame.codecOutputBuffers.size() == 0) {
    784         // No single codec output buffer has been generated. Continue to
    785         // wait.
    786         return OK;
    787     }
    788 
    789     auto res = outputANW->dequeueBuffer(mOutputSurface.get(), &inputFrame.anb, &inputFrame.fenceFd);
    790     if (res != OK) {
    791         ALOGE("%s: Error retrieving output buffer: %s (%d)", __FUNCTION__, strerror(-res),
    792                 res);
    793         return res;
    794     }
    795 
    796     // Combine current thread id, stream id and timestamp to uniquely identify image.
    797     std::ostringstream tempOutputFile;
    798     tempOutputFile << "HEIF-" << pthread_self() << "-"
    799             << getStreamId() << "-" << timestamp;
    800     inputFrame.fileFd = syscall(__NR_memfd_create, tempOutputFile.str().c_str(), MFD_CLOEXEC);
    801     if (inputFrame.fileFd < 0) {
    802         ALOGE("%s: Failed to create file %s. Error no is %d", __FUNCTION__,
    803                 tempOutputFile.str().c_str(), errno);
    804         return NO_INIT;
    805     }
    806     inputFrame.muxer = new MediaMuxer(inputFrame.fileFd, MediaMuxer::OUTPUT_FORMAT_HEIF);
    807     if (inputFrame.muxer == nullptr) {
    808         ALOGE("%s: Failed to create MediaMuxer for file fd %d",
    809                 __FUNCTION__, inputFrame.fileFd);
    810         return NO_INIT;
    811     }
    812 
    813     res = inputFrame.muxer->setOrientationHint(inputFrame.orientation);
    814     if (res != OK) {
    815         ALOGE("%s: Failed to setOrientationHint: %s (%d)", __FUNCTION__,
    816                 strerror(-res), res);
    817         return res;
    818     }
    819     // Set encoder quality
    820     {
    821         sp<AMessage> qualityParams = new AMessage;
    822         qualityParams->setInt32(PARAMETER_KEY_VIDEO_BITRATE, inputFrame.quality);
    823         res = mCodec->setParameters(qualityParams);
    824         if (res != OK) {
    825             ALOGE("%s: Failed to set codec quality: %s (%d)",
    826                     __FUNCTION__, strerror(-res), res);
    827             return res;
    828         }
    829     }
    830 
    831     ssize_t trackId = inputFrame.muxer->addTrack(mFormat);
    832     if (trackId < 0) {
    833         ALOGE("%s: Failed to addTrack to the muxer: %zd", __FUNCTION__, trackId);
    834         return NO_INIT;
    835     }
    836 
    837     inputFrame.trackIndex = trackId;
    838     inputFrame.pendingOutputTiles = mNumOutputTiles;
    839 
    840     res = inputFrame.muxer->start();
    841     if (res != OK) {
    842         ALOGE("%s: Failed to start MediaMuxer: %s (%d)",
    843                 __FUNCTION__, strerror(-res), res);
    844         return res;
    845     }
    846 
    847     return OK;
    848 }
    849 
    850 status_t HeicCompositeStream::processAppSegment(nsecs_t timestamp, InputFrame &inputFrame) {
    851     size_t app1Size = 0;
    852     auto appSegmentSize = findAppSegmentsSize(inputFrame.appSegmentBuffer.data,
    853             inputFrame.appSegmentBuffer.width * inputFrame.appSegmentBuffer.height,
    854             &app1Size);
    855     ALOGV("%s: appSegmentSize is %zu, width %d, height %d, app1Size %zu", __FUNCTION__,
    856           appSegmentSize, inputFrame.appSegmentBuffer.width,
    857           inputFrame.appSegmentBuffer.height, app1Size);
    858     if (appSegmentSize == 0) {
    859         ALOGE("%s: Failed to find JPEG APP segment size", __FUNCTION__);
    860         return NO_INIT;
    861     }
    862 
    863     std::unique_ptr<ExifUtils> exifUtils(ExifUtils::create());
    864     auto exifRes = exifUtils->initialize(inputFrame.appSegmentBuffer.data, app1Size);
    865     if (!exifRes) {
    866         ALOGE("%s: Failed to initialize ExifUtils object!", __FUNCTION__);
    867         return BAD_VALUE;
    868     }
    869     exifRes = exifUtils->setFromMetadata(*inputFrame.result, mStaticInfo,
    870             mOutputWidth, mOutputHeight);
    871     if (!exifRes) {
    872         ALOGE("%s: Failed to set Exif tags using metadata and main image sizes", __FUNCTION__);
    873         return BAD_VALUE;
    874     }
    875     exifRes = exifUtils->setOrientation(inputFrame.orientation);
    876     if (!exifRes) {
    877         ALOGE("%s: ExifUtils failed to set orientation", __FUNCTION__);
    878         return BAD_VALUE;
    879     }
    880     exifRes = exifUtils->generateApp1();
    881     if (!exifRes) {
    882         ALOGE("%s: ExifUtils failed to generate APP1 segment", __FUNCTION__);
    883         return BAD_VALUE;
    884     }
    885 
    886     unsigned int newApp1Length = exifUtils->getApp1Length();
    887     const uint8_t *newApp1Segment = exifUtils->getApp1Buffer();
    888 
    889     //Assemble the APP1 marker buffer required by MediaCodec
    890     uint8_t kExifApp1Marker[] = {'E', 'x', 'i', 'f', 0xFF, 0xE1, 0x00, 0x00};
    891     kExifApp1Marker[6] = static_cast<uint8_t>(newApp1Length >> 8);
    892     kExifApp1Marker[7] = static_cast<uint8_t>(newApp1Length & 0xFF);
    893     size_t appSegmentBufferSize = sizeof(kExifApp1Marker) +
    894             appSegmentSize - app1Size + newApp1Length;
    895     uint8_t* appSegmentBuffer = new uint8_t[appSegmentBufferSize];
    896     memcpy(appSegmentBuffer, kExifApp1Marker, sizeof(kExifApp1Marker));
    897     memcpy(appSegmentBuffer + sizeof(kExifApp1Marker), newApp1Segment, newApp1Length);
    898     if (appSegmentSize - app1Size > 0) {
    899         memcpy(appSegmentBuffer + sizeof(kExifApp1Marker) + newApp1Length,
    900                 inputFrame.appSegmentBuffer.data + app1Size, appSegmentSize - app1Size);
    901     }
    902 
    903     sp<ABuffer> aBuffer = new ABuffer(appSegmentBuffer, appSegmentBufferSize);
    904     auto res = inputFrame.muxer->writeSampleData(aBuffer, inputFrame.trackIndex,
    905             timestamp, MediaCodec::BUFFER_FLAG_MUXER_DATA);
    906     delete[] appSegmentBuffer;
    907 
    908     if (res != OK) {
    909         ALOGE("%s: Failed to write JPEG APP segments to muxer: %s (%d)",
    910                 __FUNCTION__, strerror(-res), res);
    911         return res;
    912     }
    913     inputFrame.appSegmentWritten = true;
    914 
    915     return OK;
    916 }
    917 
    918 status_t HeicCompositeStream::processCodecInputFrame(InputFrame &inputFrame) {
    919     for (auto& inputBuffer : inputFrame.codecInputBuffers) {
    920         sp<MediaCodecBuffer> buffer;
    921         auto res = mCodec->getInputBuffer(inputBuffer.index, &buffer);
    922         if (res != OK) {
    923             ALOGE("%s: Error getting codec input buffer: %s (%d)", __FUNCTION__,
    924                     strerror(-res), res);
    925             return res;
    926         }
    927 
    928         // Copy one tile from source to destination.
    929         size_t tileX = inputBuffer.tileIndex % mGridCols;
    930         size_t tileY = inputBuffer.tileIndex / mGridCols;
    931         size_t top = mGridHeight * tileY;
    932         size_t left = mGridWidth * tileX;
    933         size_t width = (tileX == static_cast<size_t>(mGridCols) - 1) ?
    934                 mOutputWidth - tileX * mGridWidth : mGridWidth;
    935         size_t height = (tileY == static_cast<size_t>(mGridRows) - 1) ?
    936                 mOutputHeight - tileY * mGridHeight : mGridHeight;
    937         ALOGV("%s: inputBuffer tileIndex [%zu, %zu], top %zu, left %zu, width %zu, height %zu",
    938                 __FUNCTION__, tileX, tileY, top, left, width, height);
    939 
    940         res = copyOneYuvTile(buffer, inputFrame.yuvBuffer, top, left, width, height);
    941         if (res != OK) {
    942             ALOGE("%s: Failed to copy YUV tile %s (%d)", __FUNCTION__,
    943                     strerror(-res), res);
    944             return res;
    945         }
    946 
    947         res = mCodec->queueInputBuffer(inputBuffer.index, 0, buffer->capacity(),
    948                 inputBuffer.timeUs, 0, nullptr /*errorDetailMsg*/);
    949         if (res != OK) {
    950             ALOGE("%s: Failed to queueInputBuffer to Codec: %s (%d)",
    951                     __FUNCTION__, strerror(-res), res);
    952             return res;
    953         }
    954     }
    955 
    956     inputFrame.codecInputBuffers.clear();
    957     return OK;
    958 }
    959 
    960 status_t HeicCompositeStream::processOneCodecOutputFrame(nsecs_t timestamp,
    961         InputFrame &inputFrame) {
    962     auto it = inputFrame.codecOutputBuffers.begin();
    963     sp<MediaCodecBuffer> buffer;
    964     status_t res = mCodec->getOutputBuffer(it->index, &buffer);
    965     if (res != OK) {
    966         ALOGE("%s: Error getting Heic codec output buffer at index %d: %s (%d)",
    967                 __FUNCTION__, it->index, strerror(-res), res);
    968         return res;
    969     }
    970     if (buffer == nullptr) {
    971         ALOGE("%s: Invalid Heic codec output buffer at index %d",
    972                 __FUNCTION__, it->index);
    973         return BAD_VALUE;
    974     }
    975 
    976     sp<ABuffer> aBuffer = new ABuffer(buffer->data(), buffer->size());
    977     res = inputFrame.muxer->writeSampleData(
    978             aBuffer, inputFrame.trackIndex, timestamp, 0 /*flags*/);
    979     if (res != OK) {
    980         ALOGE("%s: Failed to write buffer index %d to muxer: %s (%d)",
    981                 __FUNCTION__, it->index, strerror(-res), res);
    982         return res;
    983     }
    984 
    985     mCodec->releaseOutputBuffer(it->index);
    986     if (inputFrame.pendingOutputTiles == 0) {
    987         ALOGW("%s: Codec generated more tiles than expected!", __FUNCTION__);
    988     } else {
    989         inputFrame.pendingOutputTiles--;
    990     }
    991 
    992     inputFrame.codecOutputBuffers.erase(inputFrame.codecOutputBuffers.begin());
    993     return OK;
    994 }
    995 
    996 status_t HeicCompositeStream::processCompletedInputFrame(nsecs_t timestamp,
    997         InputFrame &inputFrame) {
    998     sp<ANativeWindow> outputANW = mOutputSurface;
    999     inputFrame.muxer->stop();
   1000 
   1001     // Copy the content of the file to memory.
   1002     sp<GraphicBuffer> gb = GraphicBuffer::from(inputFrame.anb);
   1003     void* dstBuffer;
   1004     auto res = gb->lockAsync(GRALLOC_USAGE_SW_WRITE_OFTEN, &dstBuffer, inputFrame.fenceFd);
   1005     if (res != OK) {
   1006         ALOGE("%s: Error trying to lock output buffer fence: %s (%d)", __FUNCTION__,
   1007                 strerror(-res), res);
   1008         return res;
   1009     }
   1010 
   1011     off_t fSize = lseek(inputFrame.fileFd, 0, SEEK_END);
   1012     if (static_cast<size_t>(fSize) > mMaxHeicBufferSize - sizeof(CameraBlob)) {
   1013         ALOGE("%s: Error: MediaMuxer output size %ld is larger than buffer sizer %zu",
   1014                 __FUNCTION__, fSize, mMaxHeicBufferSize - sizeof(CameraBlob));
   1015         return BAD_VALUE;
   1016     }
   1017 
   1018     lseek(inputFrame.fileFd, 0, SEEK_SET);
   1019     ssize_t bytesRead = read(inputFrame.fileFd, dstBuffer, fSize);
   1020     if (bytesRead < fSize) {
   1021         ALOGE("%s: Only %zd of %ld bytes read", __FUNCTION__, bytesRead, fSize);
   1022         return BAD_VALUE;
   1023     }
   1024 
   1025     close(inputFrame.fileFd);
   1026     inputFrame.fileFd = -1;
   1027 
   1028     // Fill in HEIC header
   1029     uint8_t *header = static_cast<uint8_t*>(dstBuffer) + mMaxHeicBufferSize - sizeof(CameraBlob);
   1030     struct CameraBlob *blobHeader = (struct CameraBlob *)header;
   1031     // Must be in sync with CAMERA3_HEIC_BLOB_ID in android_media_Utils.cpp
   1032     blobHeader->blobId = static_cast<CameraBlobId>(0x00FE);
   1033     blobHeader->blobSize = fSize;
   1034 
   1035     res = native_window_set_buffers_timestamp(mOutputSurface.get(), timestamp);
   1036     if (res != OK) {
   1037         ALOGE("%s: Stream %d: Error setting timestamp: %s (%d)",
   1038                __FUNCTION__, getStreamId(), strerror(-res), res);
   1039         return res;
   1040     }
   1041 
   1042     res = outputANW->queueBuffer(mOutputSurface.get(), inputFrame.anb, /*fence*/ -1);
   1043     if (res != OK) {
   1044         ALOGE("%s: Failed to queueBuffer to Heic stream: %s (%d)", __FUNCTION__,
   1045                 strerror(-res), res);
   1046         return res;
   1047     }
   1048     inputFrame.anb = nullptr;
   1049 
   1050     ATRACE_ASYNC_END("HEIC capture", inputFrame.frameNumber);
   1051     return OK;
   1052 }
   1053 
   1054 
   1055 void HeicCompositeStream::releaseInputFrameLocked(InputFrame *inputFrame /*out*/) {
   1056     if (inputFrame == nullptr) {
   1057         return;
   1058     }
   1059 
   1060     if (inputFrame->appSegmentBuffer.data != nullptr) {
   1061         mAppSegmentConsumer->unlockBuffer(inputFrame->appSegmentBuffer);
   1062         inputFrame->appSegmentBuffer.data = nullptr;
   1063         mAppSegmentBufferAcquired = false;
   1064     }
   1065 
   1066     while (!inputFrame->codecOutputBuffers.empty()) {
   1067         auto it = inputFrame->codecOutputBuffers.begin();
   1068         ALOGV("%s: releaseOutputBuffer index %d", __FUNCTION__, it->index);
   1069         mCodec->releaseOutputBuffer(it->index);
   1070         inputFrame->codecOutputBuffers.erase(it);
   1071     }
   1072 
   1073     if (inputFrame->yuvBuffer.data != nullptr) {
   1074         mMainImageConsumer->unlockBuffer(inputFrame->yuvBuffer);
   1075         inputFrame->yuvBuffer.data = nullptr;
   1076         mYuvBufferAcquired = false;
   1077     }
   1078 
   1079     while (!inputFrame->codecInputBuffers.empty()) {
   1080         auto it = inputFrame->codecInputBuffers.begin();
   1081         inputFrame->codecInputBuffers.erase(it);
   1082     }
   1083 
   1084     if ((inputFrame->error || mErrorState) && !inputFrame->errorNotified) {
   1085         notifyError(inputFrame->frameNumber);
   1086         inputFrame->errorNotified = true;
   1087     }
   1088 
   1089     if (inputFrame->fileFd >= 0) {
   1090         close(inputFrame->fileFd);
   1091         inputFrame->fileFd = -1;
   1092     }
   1093 
   1094     if (inputFrame->anb != nullptr) {
   1095         sp<ANativeWindow> outputANW = mOutputSurface;
   1096         outputANW->cancelBuffer(mOutputSurface.get(), inputFrame->anb, /*fence*/ -1);
   1097         inputFrame->anb = nullptr;
   1098     }
   1099 }
   1100 
   1101 void HeicCompositeStream::releaseInputFramesLocked(int64_t currentTs) {
   1102     auto it = mPendingInputFrames.begin();
   1103     while (it != mPendingInputFrames.end()) {
   1104         if (it->first <= currentTs) {
   1105             releaseInputFrameLocked(&it->second);
   1106             it = mPendingInputFrames.erase(it);
   1107         } else {
   1108             it++;
   1109         }
   1110     }
   1111 }
   1112 
   1113 status_t HeicCompositeStream::initializeCodec(uint32_t width, uint32_t height,
   1114         const sp<CameraDeviceBase>& cameraDevice) {
   1115     ALOGV("%s", __FUNCTION__);
   1116 
   1117     bool useGrid = false;
   1118     AString hevcName;
   1119     bool isSizeSupported = isSizeSupportedByHeifEncoder(width, height,
   1120             &mUseHeic, &useGrid, nullptr, &hevcName);
   1121     if (!isSizeSupported) {
   1122         ALOGE("%s: Encoder doesnt' support size %u x %u!",
   1123                 __FUNCTION__, width, height);
   1124         return BAD_VALUE;
   1125     }
   1126 
   1127     // Create Looper for MediaCodec.
   1128     auto desiredMime = mUseHeic ? MIMETYPE_IMAGE_ANDROID_HEIC : MIMETYPE_VIDEO_HEVC;
   1129     mCodecLooper = new ALooper;
   1130     mCodecLooper->setName("Camera3-HeicComposite-MediaCodecLooper");
   1131     status_t res = mCodecLooper->start(
   1132             false,   // runOnCallingThread
   1133             false,    // canCallJava
   1134             PRIORITY_AUDIO);
   1135     if (res != OK) {
   1136         ALOGE("%s: Failed to start codec looper: %s (%d)",
   1137                 __FUNCTION__, strerror(-res), res);
   1138         return NO_INIT;
   1139     }
   1140 
   1141     // Create HEIC/HEVC codec.
   1142     if (mUseHeic) {
   1143         mCodec = MediaCodec::CreateByType(mCodecLooper, desiredMime, true /*encoder*/);
   1144     } else {
   1145         mCodec = MediaCodec::CreateByComponentName(mCodecLooper, hevcName);
   1146     }
   1147     if (mCodec == nullptr) {
   1148         ALOGE("%s: Failed to create codec for %s", __FUNCTION__, desiredMime);
   1149         return NO_INIT;
   1150     }
   1151 
   1152     // Create Looper and handler for Codec callback.
   1153     mCodecCallbackHandler = new CodecCallbackHandler(this);
   1154     if (mCodecCallbackHandler == nullptr) {
   1155         ALOGE("%s: Failed to create codec callback handler", __FUNCTION__);
   1156         return NO_MEMORY;
   1157     }
   1158     mCallbackLooper = new ALooper;
   1159     mCallbackLooper->setName("Camera3-HeicComposite-MediaCodecCallbackLooper");
   1160     res = mCallbackLooper->start(
   1161             false,   // runOnCallingThread
   1162             false,    // canCallJava
   1163             PRIORITY_AUDIO);
   1164     if (res != OK) {
   1165         ALOGE("%s: Failed to start media callback looper: %s (%d)",
   1166                 __FUNCTION__, strerror(-res), res);
   1167         return NO_INIT;
   1168     }
   1169     mCallbackLooper->registerHandler(mCodecCallbackHandler);
   1170 
   1171     mAsyncNotify = new AMessage(kWhatCallbackNotify, mCodecCallbackHandler);
   1172     res = mCodec->setCallback(mAsyncNotify);
   1173     if (res != OK) {
   1174         ALOGE("%s: Failed to set MediaCodec callback: %s (%d)", __FUNCTION__,
   1175                 strerror(-res), res);
   1176         return res;
   1177     }
   1178 
   1179     // Create output format and configure the Codec.
   1180     sp<AMessage> outputFormat = new AMessage();
   1181     outputFormat->setString(KEY_MIME, desiredMime);
   1182     outputFormat->setInt32(KEY_BITRATE_MODE, BITRATE_MODE_CQ);
   1183     outputFormat->setInt32(KEY_QUALITY, kDefaultJpegQuality);
   1184     // Ask codec to skip timestamp check and encode all frames.
   1185     outputFormat->setInt64(KEY_MAX_PTS_GAP_TO_ENCODER, kNoFrameDropMaxPtsGap);
   1186 
   1187     int32_t gridWidth, gridHeight, gridRows, gridCols;
   1188     if (useGrid || mUseHeic) {
   1189         gridWidth = HeicEncoderInfoManager::kGridWidth;
   1190         gridHeight = HeicEncoderInfoManager::kGridHeight;
   1191         gridRows = (height + gridHeight - 1)/gridHeight;
   1192         gridCols = (width + gridWidth - 1)/gridWidth;
   1193 
   1194         if (mUseHeic) {
   1195             outputFormat->setInt32(KEY_TILE_WIDTH, gridWidth);
   1196             outputFormat->setInt32(KEY_TILE_HEIGHT, gridHeight);
   1197             outputFormat->setInt32(KEY_GRID_COLUMNS, gridCols);
   1198             outputFormat->setInt32(KEY_GRID_ROWS, gridRows);
   1199         }
   1200 
   1201     } else {
   1202         gridWidth = width;
   1203         gridHeight = height;
   1204         gridRows = 1;
   1205         gridCols = 1;
   1206     }
   1207 
   1208     outputFormat->setInt32(KEY_WIDTH, !useGrid ? width : gridWidth);
   1209     outputFormat->setInt32(KEY_HEIGHT, !useGrid ? height : gridHeight);
   1210     outputFormat->setInt32(KEY_I_FRAME_INTERVAL, 0);
   1211     outputFormat->setInt32(KEY_COLOR_FORMAT,
   1212             useGrid ? COLOR_FormatYUV420Flexible : COLOR_FormatSurface);
   1213     outputFormat->setInt32(KEY_FRAME_RATE, gridRows * gridCols);
   1214     // This only serves as a hint to encoder when encoding is not real-time.
   1215     outputFormat->setInt32(KEY_OPERATING_RATE, useGrid ? kGridOpRate : kNoGridOpRate);
   1216 
   1217     res = mCodec->configure(outputFormat, nullptr /*nativeWindow*/,
   1218             nullptr /*crypto*/, CONFIGURE_FLAG_ENCODE);
   1219     if (res != OK) {
   1220         ALOGE("%s: Failed to configure codec: %s (%d)", __FUNCTION__,
   1221                 strerror(-res), res);
   1222         return res;
   1223     }
   1224 
   1225     mGridWidth = gridWidth;
   1226     mGridHeight = gridHeight;
   1227     mGridRows = gridRows;
   1228     mGridCols = gridCols;
   1229     mUseGrid = useGrid;
   1230     mOutputWidth = width;
   1231     mOutputHeight = height;
   1232     mAppSegmentMaxSize = calcAppSegmentMaxSize(cameraDevice->info());
   1233     mMaxHeicBufferSize = mOutputWidth * mOutputHeight * 3 / 2 + mAppSegmentMaxSize;
   1234 
   1235     return OK;
   1236 }
   1237 
   1238 void HeicCompositeStream::deinitCodec() {
   1239     ALOGV("%s", __FUNCTION__);
   1240     if (mCodec != nullptr) {
   1241         mCodec->stop();
   1242         mCodec->release();
   1243         mCodec.clear();
   1244     }
   1245 
   1246     if (mCodecLooper != nullptr) {
   1247         mCodecLooper->stop();
   1248         mCodecLooper.clear();
   1249     }
   1250 
   1251     if (mCallbackLooper != nullptr) {
   1252         mCallbackLooper->stop();
   1253         mCallbackLooper.clear();
   1254     }
   1255 
   1256     mAsyncNotify.clear();
   1257     mFormat.clear();
   1258 }
   1259 
   1260 // Return the size of the complete list of app segment, 0 indicates failure
   1261 size_t HeicCompositeStream::findAppSegmentsSize(const uint8_t* appSegmentBuffer,
   1262         size_t maxSize, size_t *app1SegmentSize) {
   1263     if (appSegmentBuffer == nullptr || app1SegmentSize == nullptr) {
   1264         ALOGE("%s: Invalid input appSegmentBuffer %p, app1SegmentSize %p",
   1265                 __FUNCTION__, appSegmentBuffer, app1SegmentSize);
   1266         return 0;
   1267     }
   1268 
   1269     size_t expectedSize = 0;
   1270     // First check for EXIF transport header at the end of the buffer
   1271     const uint8_t *header = appSegmentBuffer + (maxSize - sizeof(struct CameraBlob));
   1272     const struct CameraBlob *blob = (const struct CameraBlob*)(header);
   1273     if (blob->blobId != CameraBlobId::JPEG_APP_SEGMENTS) {
   1274         ALOGE("%s: Invalid EXIF blobId %hu", __FUNCTION__, blob->blobId);
   1275         return 0;
   1276     }
   1277 
   1278     expectedSize = blob->blobSize;
   1279     if (expectedSize == 0 || expectedSize > maxSize - sizeof(struct CameraBlob)) {
   1280         ALOGE("%s: Invalid blobSize %zu.", __FUNCTION__, expectedSize);
   1281         return 0;
   1282     }
   1283 
   1284     uint32_t totalSize = 0;
   1285 
   1286     // Verify APP1 marker (mandatory)
   1287     uint8_t app1Marker[] = {0xFF, 0xE1};
   1288     if (memcmp(appSegmentBuffer, app1Marker, sizeof(app1Marker))) {
   1289         ALOGE("%s: Invalid APP1 marker: %x, %x", __FUNCTION__,
   1290                 appSegmentBuffer[0], appSegmentBuffer[1]);
   1291         return 0;
   1292     }
   1293     totalSize += sizeof(app1Marker);
   1294 
   1295     uint16_t app1Size = (static_cast<uint16_t>(appSegmentBuffer[totalSize]) << 8) +
   1296             appSegmentBuffer[totalSize+1];
   1297     totalSize += app1Size;
   1298 
   1299     ALOGV("%s: Expected APP segments size %zu, APP1 segment size %u",
   1300             __FUNCTION__, expectedSize, app1Size);
   1301     while (totalSize < expectedSize) {
   1302         if (appSegmentBuffer[totalSize] != 0xFF ||
   1303                 appSegmentBuffer[totalSize+1] <= 0xE1 ||
   1304                 appSegmentBuffer[totalSize+1] > 0xEF) {
   1305             // Invalid APPn marker
   1306             ALOGE("%s: Invalid APPn marker: %x, %x", __FUNCTION__,
   1307                     appSegmentBuffer[totalSize], appSegmentBuffer[totalSize+1]);
   1308             return 0;
   1309         }
   1310         totalSize += 2;
   1311 
   1312         uint16_t appnSize = (static_cast<uint16_t>(appSegmentBuffer[totalSize]) << 8) +
   1313                 appSegmentBuffer[totalSize+1];
   1314         totalSize += appnSize;
   1315     }
   1316 
   1317     if (totalSize != expectedSize) {
   1318         ALOGE("%s: Invalid JPEG APP segments: totalSize %u vs expected size %zu",
   1319                 __FUNCTION__, totalSize, expectedSize);
   1320         return 0;
   1321     }
   1322 
   1323     *app1SegmentSize = app1Size + sizeof(app1Marker);
   1324     return expectedSize;
   1325 }
   1326 
   1327 int64_t HeicCompositeStream::findTimestampInNsLocked(int64_t timeInUs) {
   1328     for (const auto& fn : mFrameNumberMap) {
   1329         if (timeInUs == ns2us(fn.second)) {
   1330             return fn.second;
   1331         }
   1332     }
   1333     for (const auto& inputFrame : mPendingInputFrames) {
   1334         if (timeInUs == ns2us(inputFrame.first)) {
   1335             return inputFrame.first;
   1336         }
   1337     }
   1338     return -1;
   1339 }
   1340 
   1341 status_t HeicCompositeStream::copyOneYuvTile(sp<MediaCodecBuffer>& codecBuffer,
   1342         const CpuConsumer::LockedBuffer& yuvBuffer,
   1343         size_t top, size_t left, size_t width, size_t height) {
   1344     ATRACE_CALL();
   1345 
   1346     // Get stride information for codecBuffer
   1347     sp<ABuffer> imageData;
   1348     if (!codecBuffer->meta()->findBuffer("image-data", &imageData)) {
   1349         ALOGE("%s: Codec input buffer is not for image data!", __FUNCTION__);
   1350         return BAD_VALUE;
   1351     }
   1352     if (imageData->size() != sizeof(MediaImage2)) {
   1353         ALOGE("%s: Invalid codec input image size %zu, expected %zu",
   1354                 __FUNCTION__, imageData->size(), sizeof(MediaImage2));
   1355         return BAD_VALUE;
   1356     }
   1357     MediaImage2* imageInfo = reinterpret_cast<MediaImage2*>(imageData->data());
   1358     if (imageInfo->mType != MediaImage2::MEDIA_IMAGE_TYPE_YUV ||
   1359             imageInfo->mBitDepth != 8 ||
   1360             imageInfo->mBitDepthAllocated != 8 ||
   1361             imageInfo->mNumPlanes != 3) {
   1362         ALOGE("%s: Invalid codec input image info: mType %d, mBitDepth %d, "
   1363                 "mBitDepthAllocated %d, mNumPlanes %d!", __FUNCTION__,
   1364                 imageInfo->mType, imageInfo->mBitDepth,
   1365                 imageInfo->mBitDepthAllocated, imageInfo->mNumPlanes);
   1366         return BAD_VALUE;
   1367     }
   1368 
   1369     ALOGV("%s: yuvBuffer chromaStep %d, chromaStride %d",
   1370             __FUNCTION__, yuvBuffer.chromaStep, yuvBuffer.chromaStride);
   1371     ALOGV("%s: U offset %u, V offset %u, U rowInc %d, V rowInc %d, U colInc %d, V colInc %d",
   1372             __FUNCTION__, imageInfo->mPlane[MediaImage2::U].mOffset,
   1373             imageInfo->mPlane[MediaImage2::V].mOffset,
   1374             imageInfo->mPlane[MediaImage2::U].mRowInc,
   1375             imageInfo->mPlane[MediaImage2::V].mRowInc,
   1376             imageInfo->mPlane[MediaImage2::U].mColInc,
   1377             imageInfo->mPlane[MediaImage2::V].mColInc);
   1378 
   1379     // Y
   1380     for (auto row = top; row < top+height; row++) {
   1381         uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[MediaImage2::Y].mOffset +
   1382                 imageInfo->mPlane[MediaImage2::Y].mRowInc * (row - top);
   1383         mFnCopyRow(yuvBuffer.data+row*yuvBuffer.stride+left, dst, width);
   1384     }
   1385 
   1386     // U is Cb, V is Cr
   1387     bool codecUPlaneFirst = imageInfo->mPlane[MediaImage2::V].mOffset >
   1388             imageInfo->mPlane[MediaImage2::U].mOffset;
   1389     uint32_t codecUvOffsetDiff = codecUPlaneFirst ?
   1390             imageInfo->mPlane[MediaImage2::V].mOffset - imageInfo->mPlane[MediaImage2::U].mOffset :
   1391             imageInfo->mPlane[MediaImage2::U].mOffset - imageInfo->mPlane[MediaImage2::V].mOffset;
   1392     bool isCodecUvSemiplannar = (codecUvOffsetDiff == 1) &&
   1393             (imageInfo->mPlane[MediaImage2::U].mRowInc ==
   1394             imageInfo->mPlane[MediaImage2::V].mRowInc) &&
   1395             (imageInfo->mPlane[MediaImage2::U].mColInc == 2) &&
   1396             (imageInfo->mPlane[MediaImage2::V].mColInc == 2);
   1397     bool isCodecUvPlannar =
   1398             ((codecUPlaneFirst && codecUvOffsetDiff >=
   1399                     imageInfo->mPlane[MediaImage2::U].mRowInc * imageInfo->mHeight/2) ||
   1400             ((!codecUPlaneFirst && codecUvOffsetDiff >=
   1401                     imageInfo->mPlane[MediaImage2::V].mRowInc * imageInfo->mHeight/2))) &&
   1402             imageInfo->mPlane[MediaImage2::U].mColInc == 1 &&
   1403             imageInfo->mPlane[MediaImage2::V].mColInc == 1;
   1404     bool cameraUPlaneFirst = yuvBuffer.dataCr > yuvBuffer.dataCb;
   1405 
   1406     if (isCodecUvSemiplannar && yuvBuffer.chromaStep == 2 &&
   1407             (codecUPlaneFirst == cameraUPlaneFirst)) {
   1408         // UV semiplannar
   1409         // The chrome plane could be either Cb first, or Cr first. Take the
   1410         // smaller address.
   1411         uint8_t *src = std::min(yuvBuffer.dataCb, yuvBuffer.dataCr);
   1412         MediaImage2::PlaneIndex dstPlane = codecUvOffsetDiff > 0 ? MediaImage2::U : MediaImage2::V;
   1413         for (auto row = top/2; row < (top+height)/2; row++) {
   1414             uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[dstPlane].mOffset +
   1415                     imageInfo->mPlane[dstPlane].mRowInc * (row - top/2);
   1416             mFnCopyRow(src+row*yuvBuffer.chromaStride+left, dst, width);
   1417         }
   1418     } else if (isCodecUvPlannar && yuvBuffer.chromaStep == 1) {
   1419         // U plane
   1420         for (auto row = top/2; row < (top+height)/2; row++) {
   1421             uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[MediaImage2::U].mOffset +
   1422                     imageInfo->mPlane[MediaImage2::U].mRowInc * (row - top/2);
   1423             mFnCopyRow(yuvBuffer.dataCb+row*yuvBuffer.chromaStride+left/2, dst, width/2);
   1424         }
   1425 
   1426         // V plane
   1427         for (auto row = top/2; row < (top+height)/2; row++) {
   1428             uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[MediaImage2::V].mOffset +
   1429                     imageInfo->mPlane[MediaImage2::V].mRowInc * (row - top/2);
   1430             mFnCopyRow(yuvBuffer.dataCr+row*yuvBuffer.chromaStride+left/2, dst, width/2);
   1431         }
   1432     } else {
   1433         // Convert between semiplannar and plannar, or when UV orders are
   1434         // different.
   1435         uint8_t *dst = codecBuffer->data();
   1436         for (auto row = top/2; row < (top+height)/2; row++) {
   1437             for (auto col = left/2; col < (left+width)/2; col++) {
   1438                 // U/Cb
   1439                 int32_t dstIndex = imageInfo->mPlane[MediaImage2::U].mOffset +
   1440                         imageInfo->mPlane[MediaImage2::U].mRowInc * (row - top/2) +
   1441                         imageInfo->mPlane[MediaImage2::U].mColInc * (col - left/2);
   1442                 int32_t srcIndex = row * yuvBuffer.chromaStride + yuvBuffer.chromaStep * col;
   1443                 dst[dstIndex] = yuvBuffer.dataCb[srcIndex];
   1444 
   1445                 // V/Cr
   1446                 dstIndex = imageInfo->mPlane[MediaImage2::V].mOffset +
   1447                         imageInfo->mPlane[MediaImage2::V].mRowInc * (row - top/2) +
   1448                         imageInfo->mPlane[MediaImage2::V].mColInc * (col - left/2);
   1449                 srcIndex = row * yuvBuffer.chromaStride + yuvBuffer.chromaStep * col;
   1450                 dst[dstIndex] = yuvBuffer.dataCr[srcIndex];
   1451             }
   1452         }
   1453     }
   1454     return OK;
   1455 }
   1456 
   1457 void HeicCompositeStream::initCopyRowFunction(int32_t width)
   1458 {
   1459     using namespace libyuv;
   1460 
   1461     mFnCopyRow = CopyRow_C;
   1462 #if defined(HAS_COPYROW_SSE2)
   1463     if (TestCpuFlag(kCpuHasSSE2)) {
   1464         mFnCopyRow = IS_ALIGNED(width, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2;
   1465     }
   1466 #endif
   1467 #if defined(HAS_COPYROW_AVX)
   1468     if (TestCpuFlag(kCpuHasAVX)) {
   1469         mFnCopyRow = IS_ALIGNED(width, 64) ? CopyRow_AVX : CopyRow_Any_AVX;
   1470     }
   1471 #endif
   1472 #if defined(HAS_COPYROW_ERMS)
   1473     if (TestCpuFlag(kCpuHasERMS)) {
   1474         mFnCopyRow = CopyRow_ERMS;
   1475     }
   1476 #endif
   1477 #if defined(HAS_COPYROW_NEON)
   1478     if (TestCpuFlag(kCpuHasNEON)) {
   1479         mFnCopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON;
   1480     }
   1481 #endif
   1482 #if defined(HAS_COPYROW_MIPS)
   1483     if (TestCpuFlag(kCpuHasMIPS)) {
   1484         mFnCopyRow = CopyRow_MIPS;
   1485     }
   1486 #endif
   1487 }
   1488 
   1489 size_t HeicCompositeStream::calcAppSegmentMaxSize(const CameraMetadata& info) {
   1490     camera_metadata_ro_entry_t entry = info.find(ANDROID_HEIC_INFO_MAX_JPEG_APP_SEGMENTS_COUNT);
   1491     size_t maxAppsSegment = 1;
   1492     if (entry.count > 0) {
   1493         maxAppsSegment = entry.data.u8[0] < 1 ? 1 :
   1494                 entry.data.u8[0] > 16 ? 16 : entry.data.u8[0];
   1495     }
   1496     return maxAppsSegment * (2 + 0xFFFF) + sizeof(struct CameraBlob);
   1497 }
   1498 
   1499 bool HeicCompositeStream::threadLoop() {
   1500     int64_t currentTs = INT64_MAX;
   1501     bool newInputAvailable = false;
   1502 
   1503     {
   1504         Mutex::Autolock l(mMutex);
   1505         if (mErrorState) {
   1506             // In case we landed in error state, return any pending buffers and
   1507             // halt all further processing.
   1508             compilePendingInputLocked();
   1509             releaseInputFramesLocked(currentTs);
   1510             return false;
   1511         }
   1512 
   1513 
   1514         while (!newInputAvailable) {
   1515             compilePendingInputLocked();
   1516             newInputAvailable = getNextReadyInputLocked(&currentTs);
   1517 
   1518             if (!newInputAvailable) {
   1519                 auto failingFrameNumber = getNextFailingInputLocked(&currentTs);
   1520                 if (failingFrameNumber >= 0) {
   1521                     // We cannot erase 'mPendingInputFrames[currentTs]' at this point because it is
   1522                     // possible for two internal stream buffers to fail. In such scenario the
   1523                     // composite stream should notify the client about a stream buffer error only
   1524                     // once and this information is kept within 'errorNotified'.
   1525                     // Any present failed input frames will be removed on a subsequent call to
   1526                     // 'releaseInputFramesLocked()'.
   1527                     releaseInputFrameLocked(&mPendingInputFrames[currentTs]);
   1528                     currentTs = INT64_MAX;
   1529                 }
   1530 
   1531                 auto ret = mInputReadyCondition.waitRelative(mMutex, kWaitDuration);
   1532                 if (ret == TIMED_OUT) {
   1533                     return true;
   1534                 } else if (ret != OK) {
   1535                     ALOGE("%s: Timed wait on condition failed: %s (%d)", __FUNCTION__,
   1536                             strerror(-ret), ret);
   1537                     return false;
   1538                 }
   1539             }
   1540         }
   1541     }
   1542 
   1543     auto res = processInputFrame(currentTs, mPendingInputFrames[currentTs]);
   1544     Mutex::Autolock l(mMutex);
   1545     if (res != OK) {
   1546         ALOGE("%s: Failed processing frame with timestamp: %" PRIu64 ": %s (%d)",
   1547                 __FUNCTION__, currentTs, strerror(-res), res);
   1548         mPendingInputFrames[currentTs].error = true;
   1549     }
   1550 
   1551     if (mPendingInputFrames[currentTs].error ||
   1552             (mPendingInputFrames[currentTs].appSegmentWritten &&
   1553             mPendingInputFrames[currentTs].pendingOutputTiles == 0)) {
   1554         releaseInputFramesLocked(currentTs);
   1555     }
   1556 
   1557     return true;
   1558 }
   1559 
   1560 bool HeicCompositeStream::onStreamBufferError(const CaptureResultExtras& resultExtras) {
   1561     bool res = false;
   1562     // Buffer errors concerning internal composite streams should not be directly visible to
   1563     // camera clients. They must only receive a single buffer error with the public composite
   1564     // stream id.
   1565     if ((resultExtras.errorStreamId == mAppSegmentStreamId) ||
   1566             (resultExtras.errorStreamId == mMainImageStreamId)) {
   1567         flagAnErrorFrameNumber(resultExtras.frameNumber);
   1568         res = true;
   1569     }
   1570 
   1571     return res;
   1572 }
   1573 
   1574 void HeicCompositeStream::onResultError(const CaptureResultExtras& resultExtras) {
   1575     // For result error, since the APPS_SEGMENT buffer already contains EXIF,
   1576     // simply skip using the capture result metadata to override EXIF.
   1577     Mutex::Autolock l(mMutex);
   1578 
   1579     int64_t timestamp = -1;
   1580     for (const auto& fn : mFrameNumberMap) {
   1581         if (fn.first == resultExtras.frameNumber) {
   1582             timestamp = fn.second;
   1583             break;
   1584         }
   1585     }
   1586     if (timestamp == -1) {
   1587         for (const auto& inputFrame : mPendingInputFrames) {
   1588             if (inputFrame.second.frameNumber == resultExtras.frameNumber) {
   1589                 timestamp = inputFrame.first;
   1590                 break;
   1591             }
   1592         }
   1593     }
   1594 
   1595     if (timestamp == -1) {
   1596         ALOGE("%s: Failed to find shutter timestamp for result error!", __FUNCTION__);
   1597         return;
   1598     }
   1599 
   1600     mCaptureResults.emplace(timestamp, std::make_tuple(resultExtras.frameNumber, CameraMetadata()));
   1601     mInputReadyCondition.signal();
   1602 }
   1603 
   1604 void HeicCompositeStream::CodecCallbackHandler::onMessageReceived(const sp<AMessage> &msg) {
   1605     sp<HeicCompositeStream> parent = mParent.promote();
   1606     if (parent == nullptr) return;
   1607 
   1608     switch (msg->what()) {
   1609         case kWhatCallbackNotify: {
   1610              int32_t cbID;
   1611              if (!msg->findInt32("callbackID", &cbID)) {
   1612                  ALOGE("kWhatCallbackNotify: callbackID is expected.");
   1613                  break;
   1614              }
   1615 
   1616              ALOGV("kWhatCallbackNotify: cbID = %d", cbID);
   1617 
   1618              switch (cbID) {
   1619                  case MediaCodec::CB_INPUT_AVAILABLE: {
   1620                      int32_t index;
   1621                      if (!msg->findInt32("index", &index)) {
   1622                          ALOGE("CB_INPUT_AVAILABLE: index is expected.");
   1623                          break;
   1624                      }
   1625                      parent->onHeicInputFrameAvailable(index);
   1626                      break;
   1627                  }
   1628 
   1629                  case MediaCodec::CB_OUTPUT_AVAILABLE: {
   1630                      int32_t index;
   1631                      size_t offset;
   1632                      size_t size;
   1633                      int64_t timeUs;
   1634                      int32_t flags;
   1635 
   1636                      if (!msg->findInt32("index", &index)) {
   1637                          ALOGE("CB_OUTPUT_AVAILABLE: index is expected.");
   1638                          break;
   1639                      }
   1640                      if (!msg->findSize("offset", &offset)) {
   1641                          ALOGE("CB_OUTPUT_AVAILABLE: offset is expected.");
   1642                          break;
   1643                      }
   1644                      if (!msg->findSize("size", &size)) {
   1645                          ALOGE("CB_OUTPUT_AVAILABLE: size is expected.");
   1646                          break;
   1647                      }
   1648                      if (!msg->findInt64("timeUs", &timeUs)) {
   1649                          ALOGE("CB_OUTPUT_AVAILABLE: timeUs is expected.");
   1650                          break;
   1651                      }
   1652                      if (!msg->findInt32("flags", &flags)) {
   1653                          ALOGE("CB_OUTPUT_AVAILABLE: flags is expected.");
   1654                          break;
   1655                      }
   1656 
   1657                      CodecOutputBufferInfo bufferInfo = {
   1658                          index,
   1659                          (int32_t)offset,
   1660                          (int32_t)size,
   1661                          timeUs,
   1662                          (uint32_t)flags};
   1663 
   1664                      parent->onHeicOutputFrameAvailable(bufferInfo);
   1665                      break;
   1666                  }
   1667 
   1668                  case MediaCodec::CB_OUTPUT_FORMAT_CHANGED: {
   1669                      sp<AMessage> format;
   1670                      if (!msg->findMessage("format", &format)) {
   1671                          ALOGE("CB_OUTPUT_FORMAT_CHANGED: format is expected.");
   1672                          break;
   1673                      }
   1674 
   1675                      parent->onHeicFormatChanged(format);
   1676                      break;
   1677                  }
   1678 
   1679                  case MediaCodec::CB_ERROR: {
   1680                      status_t err;
   1681                      int32_t actionCode;
   1682                      AString detail;
   1683                      if (!msg->findInt32("err", &err)) {
   1684                          ALOGE("CB_ERROR: err is expected.");
   1685                          break;
   1686                      }
   1687                      if (!msg->findInt32("action", &actionCode)) {
   1688                          ALOGE("CB_ERROR: action is expected.");
   1689                          break;
   1690                      }
   1691                      msg->findString("detail", &detail);
   1692                      ALOGE("Codec reported error(0x%x), actionCode(%d), detail(%s)",
   1693                              err, actionCode, detail.c_str());
   1694 
   1695                      parent->onHeicCodecError();
   1696                      break;
   1697                  }
   1698 
   1699                  default: {
   1700                      ALOGE("kWhatCallbackNotify: callbackID(%d) is unexpected.", cbID);
   1701                      break;
   1702                  }
   1703              }
   1704              break;
   1705         }
   1706 
   1707         default:
   1708             ALOGE("shouldn't be here");
   1709             break;
   1710     }
   1711 }
   1712 
   1713 }; // namespace camera3
   1714 }; // namespace android
   1715