Home | History | Annotate | Download | only in sfplugin 
      1 /*
      2  * Copyright (C) 2017 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #define LOG_NDEBUG 0
     18 #define LOG_TAG "CCodec"
     19 #include <utils/Log.h>
     20 
     21 #include <sstream>
     22 #include <thread>
     23 
     24 #include <C2Config.h>
     25 #include <C2Debug.h>
     26 #include <C2ParamInternal.h>
     27 #include <C2PlatformSupport.h>
     28 #include <C2V4l2Support.h>
     29 
     30 #include <android/IOMXBufferSource.h>
     31 #include <android/IGraphicBufferSource.h>
     32 #include <cutils/properties.h>
     33 #include <gui/bufferqueue/1.0/H2BGraphicBufferProducer.h>
     34 #include <gui/IGraphicBufferProducer.h>
     35 #include <gui/Surface.h>
     36 #include <media/omx/1.0/WOmx.h>
     37 #include <media/stagefright/codec2/1.0/InputSurface.h>
     38 #include <media/stagefright/BufferProducerWrapper.h>
     39 #include <media/stagefright/MediaCodecConstants.h>
     40 #include <media/stagefright/PersistentSurface.h>
     41 
     42 #include "C2OMXNode.h"
     43 #include "CCodec.h"
     44 #include "CCodecBufferChannel.h"
     45 #include "InputSurfaceWrapper.h"
     46 
     47 namespace android {
     48 
     49 using namespace std::chrono_literals;
     50 using ::android::hardware::graphics::bufferqueue::V1_0::utils::H2BGraphicBufferProducer;
     51 using BGraphicBufferSource = ::android::IGraphicBufferSource;
     52 
     53 namespace {
     54 
     55 class CCodecWatchdog : public AHandler {
     56 private:
     57     enum {
     58         kWhatRegister,
     59         kWhatWatch,
     60     };
     61     constexpr static int64_t kWatchIntervalUs = 3000000;  // 3 secs
     62 
     63 public:
     64     static sp<CCodecWatchdog> getInstance() {
     65         Mutexed<sp<CCodecWatchdog>>::Locked instance(sInstance);
     66         if (*instance == nullptr) {
     67             *instance = new CCodecWatchdog;
     68             (*instance)->init();
     69         }
     70         return *instance;
     71     }
     72 
     73     ~CCodecWatchdog() = default;
     74 
     75     void registerCodec(CCodec *codec) {
     76         sp<AMessage> msg = new AMessage(kWhatRegister, this);
     77         msg->setPointer("codec", codec);
     78         msg->post();
     79     }
     80 
     81 protected:
     82     void onMessageReceived(const sp<AMessage> &msg) {
     83         switch (msg->what()) {
     84             case kWhatRegister: {
     85                 void *ptr = nullptr;
     86                 CHECK(msg->findPointer("codec", &ptr));
     87                 Mutexed<std::list<wp<CCodec>>>::Locked codecs(mCodecs);
     88                 codecs->emplace_back((CCodec *)ptr);
     89                 break;
     90             }
     91 
     92             case kWhatWatch: {
     93                 Mutexed<std::list<wp<CCodec>>>::Locked codecs(mCodecs);
     94                 for (auto it = codecs->begin(); it != codecs->end(); ) {
     95                     sp<CCodec> codec = it->promote();
     96                     if (codec == nullptr) {
     97                         it = codecs->erase(it);
     98                         continue;
     99                     }
    100                     codec->initiateReleaseIfStuck();
    101                     ++it;
    102                 }
    103                 msg->post(kWatchIntervalUs);
    104                 break;
    105             }
    106 
    107             default: {
    108                 TRESPASS("CCodecWatchdog: unrecognized message");
    109             }
    110         }
    111     }
    112 
    113 private:
    114     CCodecWatchdog() : mLooper(new ALooper) {}
    115 
    116     void init() {
    117         mLooper->setName("CCodecWatchdog");
    118         mLooper->registerHandler(this);
    119         mLooper->start();
    120         (new AMessage(kWhatWatch, this))->post(kWatchIntervalUs);
    121     }
    122 
    123     static Mutexed<sp<CCodecWatchdog>> sInstance;
    124 
    125     sp<ALooper> mLooper;
    126     Mutexed<std::list<wp<CCodec>>> mCodecs;
    127 };
    128 
    129 Mutexed<sp<CCodecWatchdog>> CCodecWatchdog::sInstance;
    130 
    131 class C2InputSurfaceWrapper : public InputSurfaceWrapper {
    132 public:
    133     explicit C2InputSurfaceWrapper(
    134             const std::shared_ptr<Codec2Client::InputSurface> &surface) :
    135         mSurface(surface) {
    136     }
    137 
    138     ~C2InputSurfaceWrapper() override = default;
    139 
    140     status_t connect(const std::shared_ptr<Codec2Client::Component> &comp) override {
    141         if (mConnection != nullptr) {
    142             return ALREADY_EXISTS;
    143         }
    144         return static_cast<status_t>(
    145                 mSurface->connectToComponent(comp, &mConnection));
    146     }
    147 
    148     void disconnect() override {
    149         if (mConnection != nullptr) {
    150             mConnection->disconnect();
    151             mConnection = nullptr;
    152         }
    153     }
    154 
    155     status_t signalEndOfInputStream() override {
    156         C2InputSurfaceEosTuning eos(true);
    157         std::vector<std::unique_ptr<C2SettingResult>> failures;
    158         c2_status_t err = mSurface->getConfigurable()->config({&eos}, C2_MAY_BLOCK, &failures);
    159         if (err != C2_OK) {
    160             return UNKNOWN_ERROR;
    161         }
    162         return OK;
    163     }
    164 
    165     status_t configure(Config &config __unused) {
    166         // TODO
    167         return OK;
    168     }
    169 
    170 private:
    171     std::shared_ptr<Codec2Client::InputSurface> mSurface;
    172     std::shared_ptr<Codec2Client::InputSurfaceConnection> mConnection;
    173 };
    174 
    175 class GraphicBufferSourceWrapper : public InputSurfaceWrapper {
    176 public:
    177 //    explicit GraphicBufferSourceWrapper(const sp<BGraphicBufferSource> &source) : mSource(source) {}
    178     GraphicBufferSourceWrapper(
    179             const sp<BGraphicBufferSource> &source,
    180             uint32_t width,
    181             uint32_t height)
    182         : mSource(source), mWidth(width), mHeight(height) {}
    183     ~GraphicBufferSourceWrapper() override = default;
    184 
    185     status_t connect(const std::shared_ptr<Codec2Client::Component> &comp) override {
    186         // TODO: proper color aspect & dataspace
    187         android_dataspace dataSpace = HAL_DATASPACE_BT709;
    188 
    189         mNode = new C2OMXNode(comp);
    190         mNode->setFrameSize(mWidth, mHeight);
    191         mSource->configure(mNode, dataSpace);
    192 
    193         // TODO: configure according to intf().
    194         // TODO: initial color aspects (dataspace)
    195 
    196         sp<IOMXBufferSource> source = mNode->getSource();
    197         if (source == nullptr) {
    198             return NO_INIT;
    199         }
    200         constexpr size_t kNumSlots = 16;
    201         for (size_t i = 0; i < kNumSlots; ++i) {
    202             source->onInputBufferAdded(i);
    203         }
    204         source->onOmxExecuting();
    205         return OK;
    206     }
    207 
    208     void disconnect() override {
    209         if (mNode == nullptr) {
    210             return;
    211         }
    212         sp<IOMXBufferSource> source = mNode->getSource();
    213         if (source == nullptr) {
    214             ALOGD("GBSWrapper::disconnect: node is not configured with OMXBufferSource.");
    215             return;
    216         }
    217         source->onOmxIdle();
    218         source->onOmxLoaded();
    219         mNode.clear();
    220     }
    221 
    222     status_t GetStatus(const binder::Status &status) {
    223         status_t err = OK;
    224         if (!status.isOk()) {
    225             err = status.serviceSpecificErrorCode();
    226             if (err == OK) {
    227                 err = status.transactionError();
    228                 if (err == OK) {
    229                     // binder status failed, but there is no servie or transaction error
    230                     err = UNKNOWN_ERROR;
    231                 }
    232             }
    233         }
    234         return err;
    235     }
    236 
    237     status_t signalEndOfInputStream() override {
    238         return GetStatus(mSource->signalEndOfInputStream());
    239     }
    240 
    241     status_t configure(Config &config) {
    242         std::stringstream status;
    243         status_t err = OK;
    244 
    245         // handle each configuration granually, in case we need to handle part of the configuration
    246         // elsewhere
    247 
    248         // TRICKY: we do not unset frame delay repeating
    249         if (config.mMinFps > 0 && config.mMinFps != mConfig.mMinFps) {
    250             int64_t us = 1e6 / config.mMinFps + 0.5;
    251             status_t res = GetStatus(mSource->setRepeatPreviousFrameDelayUs(us));
    252             status << " minFps=" << config.mMinFps << " => repeatDelayUs=" << us;
    253             if (res != OK) {
    254                 status << " (=> " << asString(res) << ")";
    255                 err = res;
    256             }
    257             mConfig.mMinFps = config.mMinFps;
    258         }
    259 
    260         // TODO: pts gap
    261 
    262         // max fps
    263         // TRICKY: we do not unset max fps to 0 unless using fixed fps
    264         if ((config.mMaxFps > 0 || (config.mFixedAdjustedFps > 0 && config.mMaxFps == 0))
    265                 && config.mMaxFps != mConfig.mMaxFps) {
    266             status_t res = GetStatus(mSource->setMaxFps(config.mMaxFps));
    267             status << " maxFps=" << config.mMaxFps;
    268             if (res != OK) {
    269                 status << " (=> " << asString(res) << ")";
    270                 err = res;
    271             }
    272             mConfig.mMaxFps = config.mMaxFps;
    273         }
    274 
    275         if (config.mTimeOffsetUs != mConfig.mTimeOffsetUs) {
    276             status_t res = GetStatus(mSource->setTimeOffsetUs(config.mTimeOffsetUs));
    277             status << " timeOffset " << config.mTimeOffsetUs << "us";
    278             if (res != OK) {
    279                 status << " (=> " << asString(res) << ")";
    280                 err = res;
    281             }
    282             mConfig.mTimeOffsetUs = config.mTimeOffsetUs;
    283         }
    284 
    285         // TODO: time lapse config
    286 
    287         if (config.mStartAtUs != mConfig.mStartAtUs
    288                 || (config.mStopped != mConfig.mStopped && !config.mStopped)) {
    289             status_t res = GetStatus(mSource->setStartTimeUs(config.mStartAtUs));
    290             status << " start at " << config.mStartAtUs << "us";
    291             if (res != OK) {
    292                 status << " (=> " << asString(res) << ")";
    293                 err = res;
    294             }
    295             mConfig.mStartAtUs = config.mStartAtUs;
    296             mConfig.mStopped = config.mStopped;
    297         }
    298 
    299         // suspend-resume
    300         if (config.mSuspended != mConfig.mSuspended) {
    301             status_t res = GetStatus(mSource->setSuspend(config.mSuspended, config.mSuspendAtUs));
    302             status << " " << (config.mSuspended ? "suspend" : "resume")
    303                     << " at " << config.mSuspendAtUs << "us";
    304             if (res != OK) {
    305                 status << " (=> " << asString(res) << ")";
    306                 err = res;
    307             }
    308             mConfig.mSuspended = config.mSuspended;
    309             mConfig.mSuspendAtUs = config.mSuspendAtUs;
    310         }
    311 
    312         if (config.mStopped != mConfig.mStopped && config.mStopped) {
    313             status_t res = GetStatus(mSource->setStopTimeUs(config.mStopAtUs));
    314             status << " stop at " << config.mStopAtUs << "us";
    315             if (res != OK) {
    316                 status << " (=> " << asString(res) << ")";
    317                 err = res;
    318             } else {
    319                 status << " delayUs";
    320                 res = GetStatus(mSource->getStopTimeOffsetUs(&config.mInputDelayUs));
    321                 if (res != OK) {
    322                     status << " (=> " << asString(res) << ")";
    323                 } else {
    324                     status << "=" << config.mInputDelayUs << "us";
    325                 }
    326                 mConfig.mInputDelayUs = config.mInputDelayUs;
    327             }
    328             mConfig.mStopAtUs = config.mStopAtUs;
    329             mConfig.mStopped = config.mStopped;
    330         }
    331 
    332         // color aspects (android._color-aspects)
    333 
    334         // consumer usage
    335         ALOGD("ISConfig%s", status.str().c_str());
    336         return err;
    337     }
    338 
    339 private:
    340     sp<BGraphicBufferSource> mSource;
    341     sp<C2OMXNode> mNode;
    342     uint32_t mWidth;
    343     uint32_t mHeight;
    344     Config mConfig;
    345 };
    346 
    347 class Codec2ClientInterfaceWrapper : public C2ComponentStore {
    348     std::shared_ptr<Codec2Client> mClient;
    349 
    350 public:
    351     Codec2ClientInterfaceWrapper(std::shared_ptr<Codec2Client> client)
    352         : mClient(client) { }
    353 
    354     virtual ~Codec2ClientInterfaceWrapper() = default;
    355 
    356     virtual c2_status_t config_sm(
    357             const std::vector<C2Param *> &params,
    358             std::vector<std::unique_ptr<C2SettingResult>> *const failures) {
    359         return mClient->config(params, C2_MAY_BLOCK, failures);
    360     };
    361 
    362     virtual c2_status_t copyBuffer(
    363             std::shared_ptr<C2GraphicBuffer>,
    364             std::shared_ptr<C2GraphicBuffer>) {
    365         return C2_OMITTED;
    366     }
    367 
    368     virtual c2_status_t createComponent(
    369             C2String, std::shared_ptr<C2Component> *const component) {
    370         component->reset();
    371         return C2_OMITTED;
    372     }
    373 
    374     virtual c2_status_t createInterface(
    375             C2String, std::shared_ptr<C2ComponentInterface> *const interface) {
    376         interface->reset();
    377         return C2_OMITTED;
    378     }
    379 
    380     virtual c2_status_t query_sm(
    381             const std::vector<C2Param *> &stackParams,
    382             const std::vector<C2Param::Index> &heapParamIndices,
    383             std::vector<std::unique_ptr<C2Param>> *const heapParams) const {
    384         return mClient->query(stackParams, heapParamIndices, C2_MAY_BLOCK, heapParams);
    385     }
    386 
    387     virtual c2_status_t querySupportedParams_nb(
    388             std::vector<std::shared_ptr<C2ParamDescriptor>> *const params) const {
    389         return mClient->querySupportedParams(params);
    390     }
    391 
    392     virtual c2_status_t querySupportedValues_sm(
    393             std::vector<C2FieldSupportedValuesQuery> &fields) const {
    394         return mClient->querySupportedValues(fields, C2_MAY_BLOCK);
    395     }
    396 
    397     virtual C2String getName() const {
    398         return mClient->getName();
    399     }
    400 
    401     virtual std::shared_ptr<C2ParamReflector> getParamReflector() const {
    402         return mClient->getParamReflector();
    403     }
    404 
    405     virtual std::vector<std::shared_ptr<const C2Component::Traits>> listComponents() {
    406         return std::vector<std::shared_ptr<const C2Component::Traits>>();
    407     }
    408 };
    409 
    410 }  // namespace
    411 
    412 // CCodec::ClientListener
    413 
    414 struct CCodec::ClientListener : public Codec2Client::Listener {
    415 
    416     explicit ClientListener(const wp<CCodec> &codec) : mCodec(codec) {}
    417 
    418     virtual void onWorkDone(
    419             const std::weak_ptr<Codec2Client::Component>& component,
    420             std::list<std::unique_ptr<C2Work>>& workItems) override {
    421         (void)component;
    422         sp<CCodec> codec(mCodec.promote());
    423         if (!codec) {
    424             return;
    425         }
    426         codec->onWorkDone(workItems);
    427     }
    428 
    429     virtual void onTripped(
    430             const std::weak_ptr<Codec2Client::Component>& component,
    431             const std::vector<std::shared_ptr<C2SettingResult>>& settingResult
    432             ) override {
    433         // TODO
    434         (void)component;
    435         (void)settingResult;
    436     }
    437 
    438     virtual void onError(
    439             const std::weak_ptr<Codec2Client::Component>& component,
    440             uint32_t errorCode) override {
    441         // TODO
    442         (void)component;
    443         (void)errorCode;
    444     }
    445 
    446     virtual void onDeath(
    447             const std::weak_ptr<Codec2Client::Component>& component) override {
    448         { // Log the death of the component.
    449             std::shared_ptr<Codec2Client::Component> comp = component.lock();
    450             if (!comp) {
    451                 ALOGE("Codec2 component died.");
    452             } else {
    453                 ALOGE("Codec2 component \"%s\" died.", comp->getName().c_str());
    454             }
    455         }
    456 
    457         // Report to MediaCodec.
    458         sp<CCodec> codec(mCodec.promote());
    459         if (!codec || !codec->mCallback) {
    460             return;
    461         }
    462         codec->mCallback->onError(DEAD_OBJECT, ACTION_CODE_FATAL);
    463     }
    464 
    465     virtual void onFramesRendered(
    466             const std::vector<RenderedFrame>& renderedFrames) override {
    467         // TODO
    468         (void)renderedFrames;
    469     }
    470 
    471 private:
    472     wp<CCodec> mCodec;
    473 };
    474 
    475 // CCodecCallbackImpl
    476 
    477 class CCodecCallbackImpl : public CCodecCallback {
    478 public:
    479     explicit CCodecCallbackImpl(CCodec *codec) : mCodec(codec) {}
    480     ~CCodecCallbackImpl() override = default;
    481 
    482     void onError(status_t err, enum ActionCode actionCode) override {
    483         mCodec->mCallback->onError(err, actionCode);
    484     }
    485 
    486     void onOutputFramesRendered(int64_t mediaTimeUs, nsecs_t renderTimeNs) override {
    487         mCodec->mCallback->onOutputFramesRendered(
    488                 {RenderedFrameInfo(mediaTimeUs, renderTimeNs)});
    489     }
    490 
    491 private:
    492     CCodec *mCodec;
    493 };
    494 
    495 // CCodec
    496 
    497 CCodec::CCodec()
    498     : mChannel(new CCodecBufferChannel(std::make_shared<CCodecCallbackImpl>(this))) {
    499     CCodecWatchdog::getInstance()->registerCodec(this);
    500 }
    501 
    502 CCodec::~CCodec() {
    503 }
    504 
    505 std::shared_ptr<BufferChannelBase> CCodec::getBufferChannel() {
    506     return mChannel;
    507 }
    508 
    509 status_t CCodec::tryAndReportOnError(std::function<status_t()> job) {
    510     status_t err = job();
    511     if (err != C2_OK) {
    512         mCallback->onError(err, ACTION_CODE_FATAL);
    513     }
    514     return err;
    515 }
    516 
    517 void CCodec::initiateAllocateComponent(const sp<AMessage> &msg) {
    518     auto setAllocating = [this] {
    519         Mutexed<State>::Locked state(mState);
    520         if (state->get() != RELEASED) {
    521             return INVALID_OPERATION;
    522         }
    523         state->set(ALLOCATING);
    524         return OK;
    525     };
    526     if (tryAndReportOnError(setAllocating) != OK) {
    527         return;
    528     }
    529 
    530     sp<RefBase> codecInfo;
    531     CHECK(msg->findObject("codecInfo", &codecInfo));
    532     // For Codec 2.0 components, componentName == codecInfo->getCodecName().
    533 
    534     sp<AMessage> allocMsg(new AMessage(kWhatAllocate, this));
    535     allocMsg->setObject("codecInfo", codecInfo);
    536     allocMsg->post();
    537 }
    538 
    539 void CCodec::allocate(const sp<MediaCodecInfo> &codecInfo) {
    540     if (codecInfo == nullptr) {
    541         mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
    542         return;
    543     }
    544     ALOGV("allocate(%s)", codecInfo->getCodecName());
    545     mClientListener.reset(new ClientListener(this));
    546 
    547     AString componentName = codecInfo->getCodecName();
    548     std::shared_ptr<Codec2Client> client;
    549 
    550     // set up preferred component store to access vendor store parameters
    551     client = Codec2Client::CreateFromService("default", false);
    552     if (client) {
    553         ALOGI("setting up '%s' as default (vendor) store", client->getInstanceName().c_str());
    554         SetPreferredCodec2ComponentStore(
    555                 std::make_shared<Codec2ClientInterfaceWrapper>(client));
    556     }
    557 
    558     std::shared_ptr<Codec2Client::Component> comp =
    559             Codec2Client::CreateComponentByName(
    560             componentName.c_str(),
    561             mClientListener,
    562             &client);
    563     if (!comp) {
    564         ALOGE("Failed Create component: %s", componentName.c_str());
    565         Mutexed<State>::Locked state(mState);
    566         state->set(RELEASED);
    567         state.unlock();
    568         mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
    569         state.lock();
    570         return;
    571     }
    572     ALOGI("Created component [%s]", componentName.c_str());
    573     mChannel->setComponent(comp);
    574     auto setAllocated = [this, comp, client] {
    575         Mutexed<State>::Locked state(mState);
    576         if (state->get() != ALLOCATING) {
    577             state->set(RELEASED);
    578             return UNKNOWN_ERROR;
    579         }
    580         state->set(ALLOCATED);
    581         state->comp = comp;
    582         mClient = client;
    583         return OK;
    584     };
    585     if (tryAndReportOnError(setAllocated) != OK) {
    586         return;
    587     }
    588 
    589     // initialize config here in case setParameters is called prior to configure
    590     Mutexed<Config>::Locked config(mConfig);
    591     status_t err = config->initialize(mClient, comp);
    592     if (err != OK) {
    593         ALOGW("Failed to initialize configuration support");
    594         // TODO: report error once we complete implementation.
    595     }
    596     config->queryConfiguration(comp);
    597 
    598     mCallback->onComponentAllocated(comp->getName().c_str());
    599 }
    600 
    601 void CCodec::initiateConfigureComponent(const sp<AMessage> &format) {
    602     auto checkAllocated = [this] {
    603         Mutexed<State>::Locked state(mState);
    604         return (state->get() != ALLOCATED) ? UNKNOWN_ERROR : OK;
    605     };
    606     if (tryAndReportOnError(checkAllocated) != OK) {
    607         return;
    608     }
    609 
    610     sp<AMessage> msg(new AMessage(kWhatConfigure, this));
    611     msg->setMessage("format", format);
    612     msg->post();
    613 }
    614 
    615 void CCodec::configure(const sp<AMessage> &msg) {
    616     std::shared_ptr<Codec2Client::Component> comp;
    617     auto checkAllocated = [this, &comp] {
    618         Mutexed<State>::Locked state(mState);
    619         if (state->get() != ALLOCATED) {
    620             state->set(RELEASED);
    621             return UNKNOWN_ERROR;
    622         }
    623         comp = state->comp;
    624         return OK;
    625     };
    626     if (tryAndReportOnError(checkAllocated) != OK) {
    627         return;
    628     }
    629 
    630     auto doConfig = [msg, comp, this]() -> status_t {
    631         AString mime;
    632         if (!msg->findString("mime", &mime)) {
    633             return BAD_VALUE;
    634         }
    635 
    636         int32_t encoder;
    637         if (!msg->findInt32("encoder", &encoder)) {
    638             encoder = false;
    639         }
    640 
    641         // TODO: read from intf()
    642         if ((!encoder) != (comp->getName().find("encoder") == std::string::npos)) {
    643             return UNKNOWN_ERROR;
    644         }
    645 
    646         int32_t storeMeta;
    647         if (encoder
    648                 && msg->findInt32("android._input-metadata-buffer-type", &storeMeta)
    649                 && storeMeta != kMetadataBufferTypeInvalid) {
    650             if (storeMeta != kMetadataBufferTypeANWBuffer) {
    651                 ALOGD("Only ANW buffers are supported for legacy metadata mode");
    652                 return BAD_VALUE;
    653             }
    654             mChannel->setMetaMode(CCodecBufferChannel::MODE_ANW);
    655         }
    656 
    657         sp<RefBase> obj;
    658         sp<Surface> surface;
    659         if (msg->findObject("native-window", &obj)) {
    660             surface = static_cast<Surface *>(obj.get());
    661             setSurface(surface);
    662         }
    663 
    664         Mutexed<Config>::Locked config(mConfig);
    665 
    666         /*
    667          * Handle input surface configuration
    668          */
    669         if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))
    670                 && (config->mDomain & Config::IS_ENCODER)) {
    671             config->mISConfig.reset(new InputSurfaceWrapper::Config{});
    672             {
    673                 config->mISConfig->mMinFps = 0;
    674                 int64_t value;
    675                 if (msg->findInt64("repeat-previous-frame-after", &value) && value > 0) {
    676                     config->mISConfig->mMinFps = 1e6 / value;
    677                 }
    678                 (void)msg->findFloat("max-fps-to-encoder", &config->mISConfig->mMaxFps);
    679                 config->mISConfig->mMinAdjustedFps = 0;
    680                 config->mISConfig->mFixedAdjustedFps = 0;
    681                 if (msg->findInt64("max-pts-gap-to-encoder", &value)) {
    682                     if (value < 0 && value >= INT32_MIN) {
    683                         config->mISConfig->mFixedAdjustedFps = -1e6 / value;
    684                     } else if (value > 0 && value <= INT32_MAX) {
    685                         config->mISConfig->mMinAdjustedFps = 1e6 / value;
    686                     }
    687                 }
    688             }
    689 
    690             {
    691                 double value;
    692                 if (!msg->findDouble("time-lapse-fps", &value)) {
    693                     config->mISConfig->mCaptureFps = value;
    694                 }
    695             }
    696 
    697             {
    698                 config->mISConfig->mSuspended = false;
    699                 config->mISConfig->mSuspendAtUs = -1;
    700                 int32_t value;
    701                 if (msg->findInt32("create-input-buffers-suspended", &value) && value) {
    702                     config->mISConfig->mSuspended = true;
    703                 }
    704             }
    705         }
    706 
    707         /*
    708          * Handle desired color format.
    709          */
    710         if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))) {
    711             int32_t format = -1;
    712             if (!msg->findInt32(KEY_COLOR_FORMAT, &format)) {
    713                 /*
    714                  * Also handle default color format (encoders require color format, so this is only
    715                  * needed for decoders.
    716                  */
    717                 if (!(config->mDomain & Config::IS_ENCODER)) {
    718                     format = (surface == nullptr) ? COLOR_FormatYUV420Planar : COLOR_FormatSurface;
    719                 }
    720             }
    721 
    722             if (format >= 0) {
    723                 msg->setInt32("android._color-format", format);
    724             }
    725         }
    726 
    727         std::vector<std::unique_ptr<C2Param>> configUpdate;
    728         status_t err = config->getConfigUpdateFromSdkParams(
    729                 comp, msg, Config::CONFIG, C2_DONT_BLOCK, &configUpdate);
    730         if (err != OK) {
    731             ALOGW("failed to convert configuration to c2 params");
    732         }
    733         err = config->setParameters(comp, configUpdate, C2_DONT_BLOCK);
    734         if (err != OK) {
    735             ALOGW("failed to configure c2 params");
    736             return err;
    737         }
    738 
    739         std::vector<std::unique_ptr<C2Param>> params;
    740         C2StreamUsageTuning::input usage(0u, 0u);
    741         C2StreamMaxBufferSizeInfo::input maxInputSize(0u, 0u);
    742 
    743         std::initializer_list<C2Param::Index> indices {
    744         };
    745         c2_status_t c2err = comp->query(
    746                 { &usage, &maxInputSize },
    747                 indices,
    748                 C2_DONT_BLOCK,
    749                 &params);
    750         if (c2err != C2_OK && c2err != C2_BAD_INDEX) {
    751             ALOGE("Failed to query component interface: %d", c2err);
    752             return UNKNOWN_ERROR;
    753         }
    754         if (params.size() != indices.size()) {
    755             ALOGE("Component returns wrong number of params: expected %zu actual %zu",
    756                     indices.size(), params.size());
    757             return UNKNOWN_ERROR;
    758         }
    759         if (usage && (usage.value & C2MemoryUsage::CPU_READ)) {
    760             config->mInputFormat->setInt32("using-sw-read-often", true);
    761         }
    762 
    763         // use client specified input size if specified
    764         bool clientInputSize = msg->findInt32(KEY_MAX_INPUT_SIZE, (int32_t*)&maxInputSize.value);
    765 
    766         // TEMP: enforce minimum buffer size of 1MB for video decoders
    767         if (!clientInputSize && maxInputSize.value == 0
    768                 && !encoder && !(config->mDomain & Config::IS_AUDIO)) {
    769             maxInputSize.value = 1048576u;
    770         }
    771 
    772         // TODO: do this based on component requiring linear allocator for input
    773         if ((config->mDomain & Config::IS_DECODER) || (config->mDomain & Config::IS_AUDIO)) {
    774             // Pass max input size on input format to the buffer channel (if supplied by the
    775             // component or by a default)
    776             if (maxInputSize.value) {
    777                 config->mInputFormat->setInt32(
    778                         KEY_MAX_INPUT_SIZE,
    779                         (int32_t)(c2_min(maxInputSize.value, uint32_t(INT32_MAX))));
    780             }
    781         }
    782 
    783         if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))) {
    784             // Set desired color format from configuration parameter
    785             int32_t format;
    786             if (msg->findInt32("android._color-format", &format)) {
    787                 if (config->mDomain & Config::IS_ENCODER) {
    788                     config->mInputFormat->setInt32(KEY_COLOR_FORMAT, format);
    789                 } else {
    790                     config->mOutputFormat->setInt32(KEY_COLOR_FORMAT, format);
    791                 }
    792             }
    793         }
    794 
    795         // propagate encoder delay and padding to output format
    796         if ((config->mDomain & Config::IS_DECODER) && (config->mDomain & Config::IS_AUDIO)) {
    797             int delay = 0;
    798             if (msg->findInt32("encoder-delay", &delay)) {
    799                 config->mOutputFormat->setInt32("encoder-delay", delay);
    800             }
    801             int padding = 0;
    802             if (msg->findInt32("encoder-padding", &padding)) {
    803                 config->mOutputFormat->setInt32("encoder-padding", padding);
    804             }
    805         }
    806 
    807         ALOGD("setup formats input: %s and output: %s",
    808                 config->mInputFormat->debugString().c_str(),
    809                 config->mOutputFormat->debugString().c_str());
    810         return OK;
    811     };
    812     if (tryAndReportOnError(doConfig) != OK) {
    813         return;
    814     }
    815 
    816     Mutexed<Config>::Locked config(mConfig);
    817 
    818     mCallback->onComponentConfigured(config->mInputFormat, config->mOutputFormat);
    819 }
    820 
    821 void CCodec::initiateCreateInputSurface() {
    822     status_t err = [this] {
    823         Mutexed<State>::Locked state(mState);
    824         if (state->get() != ALLOCATED) {
    825             return UNKNOWN_ERROR;
    826         }
    827         // TODO: read it from intf() properly.
    828         if (state->comp->getName().find("encoder") == std::string::npos) {
    829             return INVALID_OPERATION;
    830         }
    831         return OK;
    832     }();
    833     if (err != OK) {
    834         mCallback->onInputSurfaceCreationFailed(err);
    835         return;
    836     }
    837 
    838     (new AMessage(kWhatCreateInputSurface, this))->post();
    839 }
    840 
    841 void CCodec::createInputSurface() {
    842     status_t err;
    843     sp<IGraphicBufferProducer> bufferProducer;
    844 
    845     sp<AMessage> inputFormat;
    846     sp<AMessage> outputFormat;
    847     {
    848         Mutexed<Config>::Locked config(mConfig);
    849         inputFormat = config->mInputFormat;
    850         outputFormat = config->mOutputFormat;
    851     }
    852 
    853     // TODO: Remove this property check and assume it's always true.
    854     if (property_get_bool("debug.stagefright.c2inputsurface", false)) {
    855         std::shared_ptr<Codec2Client::InputSurface> surface;
    856 
    857         err = static_cast<status_t>(mClient->createInputSurface(&surface));
    858         if (err != OK) {
    859             ALOGE("Failed to create input surface: %d", static_cast<int>(err));
    860             mCallback->onInputSurfaceCreationFailed(err);
    861             return;
    862         }
    863         if (!surface) {
    864             ALOGE("Failed to create input surface: null input surface");
    865             mCallback->onInputSurfaceCreationFailed(UNKNOWN_ERROR);
    866             return;
    867         }
    868         bufferProducer = surface->getGraphicBufferProducer();
    869         err = setupInputSurface(std::make_shared<C2InputSurfaceWrapper>(surface));
    870     } else { // TODO: Remove this block.
    871         using namespace ::android::hardware::media::omx::V1_0;
    872         sp<IOmx> tOmx = IOmx::getService("default");
    873         if (tOmx == nullptr) {
    874             ALOGE("Failed to create input surface");
    875             mCallback->onInputSurfaceCreationFailed(UNKNOWN_ERROR);
    876             return;
    877         }
    878         sp<IOMX> omx = new utils::LWOmx(tOmx);
    879 
    880         sp<BGraphicBufferSource> bufferSource;
    881         err = omx->createInputSurface(&bufferProducer, &bufferSource);
    882 
    883         if (err != OK) {
    884             ALOGE("Failed to create input surface: %d", err);
    885             mCallback->onInputSurfaceCreationFailed(err);
    886             return;
    887         }
    888         int32_t width = 0;
    889         (void)outputFormat->findInt32("width", &width);
    890         int32_t height = 0;
    891         (void)outputFormat->findInt32("height", &height);
    892         err = setupInputSurface(std::make_shared<GraphicBufferSourceWrapper>(
    893                 bufferSource, width, height));
    894     }
    895 
    896     if (err != OK) {
    897         ALOGE("Failed to set up input surface: %d", err);
    898         mCallback->onInputSurfaceCreationFailed(err);
    899         return;
    900     }
    901     mCallback->onInputSurfaceCreated(
    902             inputFormat,
    903             outputFormat,
    904             new BufferProducerWrapper(bufferProducer));
    905 }
    906 
    907 status_t CCodec::setupInputSurface(const std::shared_ptr<InputSurfaceWrapper> &surface) {
    908     status_t err = mChannel->setInputSurface(surface);
    909     if (err != OK) {
    910         return err;
    911     }
    912 
    913     Mutexed<Config>::Locked config(mConfig);
    914     config->mInputSurface = surface;
    915     if (config->mISConfig) {
    916         surface->configure(*config->mISConfig);
    917     } else {
    918         ALOGD("ISConfig: no configuration");
    919     }
    920 
    921     // TODO: configure |surface| with other settings.
    922     return OK;
    923 }
    924 
    925 void CCodec::initiateSetInputSurface(const sp<PersistentSurface> &surface) {
    926     sp<AMessage> msg = new AMessage(kWhatSetInputSurface, this);
    927     msg->setObject("surface", surface);
    928     msg->post();
    929 }
    930 
    931 void CCodec::setInputSurface(const sp<PersistentSurface> &surface) {
    932     sp<AMessage> inputFormat;
    933     sp<AMessage> outputFormat;
    934     {
    935         Mutexed<Config>::Locked config(mConfig);
    936         inputFormat = config->mInputFormat;
    937         outputFormat = config->mOutputFormat;
    938     }
    939     int32_t width = 0;
    940     (void)outputFormat->findInt32("width", &width);
    941     int32_t height = 0;
    942     (void)outputFormat->findInt32("height", &height);
    943     status_t err = setupInputSurface(std::make_shared<GraphicBufferSourceWrapper>(
    944             surface->getBufferSource(), width, height));
    945     if (err != OK) {
    946         ALOGE("Failed to set up input surface: %d", err);
    947         mCallback->onInputSurfaceDeclined(err);
    948         return;
    949     }
    950     mCallback->onInputSurfaceAccepted(inputFormat, outputFormat);
    951 }
    952 
    953 void CCodec::initiateStart() {
    954     auto setStarting = [this] {
    955         Mutexed<State>::Locked state(mState);
    956         if (state->get() != ALLOCATED) {
    957             return UNKNOWN_ERROR;
    958         }
    959         state->set(STARTING);
    960         return OK;
    961     };
    962     if (tryAndReportOnError(setStarting) != OK) {
    963         return;
    964     }
    965 
    966     (new AMessage(kWhatStart, this))->post();
    967 }
    968 
    969 void CCodec::start() {
    970     std::shared_ptr<Codec2Client::Component> comp;
    971     auto checkStarting = [this, &comp] {
    972         Mutexed<State>::Locked state(mState);
    973         if (state->get() != STARTING) {
    974             return UNKNOWN_ERROR;
    975         }
    976         comp = state->comp;
    977         return OK;
    978     };
    979     if (tryAndReportOnError(checkStarting) != OK) {
    980         return;
    981     }
    982 
    983     c2_status_t err = comp->start();
    984     if (err != C2_OK) {
    985         // TODO: convert err into status_t
    986         mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
    987         return;
    988     }
    989     sp<AMessage> inputFormat;
    990     sp<AMessage> outputFormat;
    991     {
    992         Mutexed<Config>::Locked config(mConfig);
    993         inputFormat = config->mInputFormat;
    994         outputFormat = config->mOutputFormat;
    995     }
    996     status_t err2 = mChannel->start(inputFormat, outputFormat);
    997     if (err2 != OK) {
    998         mCallback->onError(err2, ACTION_CODE_FATAL);
    999         return;
   1000     }
   1001 
   1002     auto setRunning = [this] {
   1003         Mutexed<State>::Locked state(mState);
   1004         if (state->get() != STARTING) {
   1005             return UNKNOWN_ERROR;
   1006         }
   1007         state->set(RUNNING);
   1008         return OK;
   1009     };
   1010     if (tryAndReportOnError(setRunning) != OK) {
   1011         return;
   1012     }
   1013     mCallback->onStartCompleted();
   1014 }
   1015 
   1016 void CCodec::initiateShutdown(bool keepComponentAllocated) {
   1017     if (keepComponentAllocated) {
   1018         initiateStop();
   1019     } else {
   1020         initiateRelease();
   1021     }
   1022 }
   1023 
   1024 void CCodec::initiateStop() {
   1025     {
   1026         Mutexed<State>::Locked state(mState);
   1027         if (state->get() == ALLOCATED
   1028                 || state->get()  == RELEASED
   1029                 || state->get() == STOPPING
   1030                 || state->get() == RELEASING) {
   1031             // We're already stopped, released, or doing it right now.
   1032             state.unlock();
   1033             mCallback->onStopCompleted();
   1034             state.lock();
   1035             return;
   1036         }
   1037         state->set(STOPPING);
   1038     }
   1039 
   1040     mChannel->stop();
   1041     (new AMessage(kWhatStop, this))->post();
   1042 }
   1043 
   1044 void CCodec::stop() {
   1045     std::shared_ptr<Codec2Client::Component> comp;
   1046     {
   1047         Mutexed<State>::Locked state(mState);
   1048         if (state->get() == RELEASING) {
   1049             state.unlock();
   1050             // We're already stopped or release is in progress.
   1051             mCallback->onStopCompleted();
   1052             state.lock();
   1053             return;
   1054         } else if (state->get() != STOPPING) {
   1055             state.unlock();
   1056             mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
   1057             state.lock();
   1058             return;
   1059         }
   1060         comp = state->comp;
   1061     }
   1062     status_t err = comp->stop();
   1063     if (err != C2_OK) {
   1064         // TODO: convert err into status_t
   1065         mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
   1066     }
   1067 
   1068     {
   1069         Mutexed<State>::Locked state(mState);
   1070         if (state->get() == STOPPING) {
   1071             state->set(ALLOCATED);
   1072         }
   1073     }
   1074     mCallback->onStopCompleted();
   1075 }
   1076 
   1077 void CCodec::initiateRelease(bool sendCallback /* = true */) {
   1078     {
   1079         Mutexed<State>::Locked state(mState);
   1080         if (state->get() == RELEASED || state->get() == RELEASING) {
   1081             // We're already released or doing it right now.
   1082             if (sendCallback) {
   1083                 state.unlock();
   1084                 mCallback->onReleaseCompleted();
   1085                 state.lock();
   1086             }
   1087             return;
   1088         }
   1089         if (state->get() == ALLOCATING) {
   1090             state->set(RELEASING);
   1091             // With the altered state allocate() would fail and clean up.
   1092             if (sendCallback) {
   1093                 state.unlock();
   1094                 mCallback->onReleaseCompleted();
   1095                 state.lock();
   1096             }
   1097             return;
   1098         }
   1099         state->set(RELEASING);
   1100     }
   1101 
   1102     mChannel->stop();
   1103     std::thread([this, sendCallback] { release(sendCallback); }).detach();
   1104 }
   1105 
   1106 void CCodec::release(bool sendCallback) {
   1107     std::shared_ptr<Codec2Client::Component> comp;
   1108     {
   1109         Mutexed<State>::Locked state(mState);
   1110         if (state->get() == RELEASED) {
   1111             if (sendCallback) {
   1112                 state.unlock();
   1113                 mCallback->onReleaseCompleted();
   1114                 state.lock();
   1115             }
   1116             return;
   1117         }
   1118         comp = state->comp;
   1119     }
   1120     comp->release();
   1121 
   1122     {
   1123         Mutexed<State>::Locked state(mState);
   1124         state->set(RELEASED);
   1125         state->comp.reset();
   1126     }
   1127     if (sendCallback) {
   1128         mCallback->onReleaseCompleted();
   1129     }
   1130 }
   1131 
   1132 status_t CCodec::setSurface(const sp<Surface> &surface) {
   1133     return mChannel->setSurface(surface);
   1134 }
   1135 
   1136 void CCodec::signalFlush() {
   1137     status_t err = [this] {
   1138         Mutexed<State>::Locked state(mState);
   1139         if (state->get() == FLUSHED) {
   1140             return ALREADY_EXISTS;
   1141         }
   1142         if (state->get() != RUNNING) {
   1143             return UNKNOWN_ERROR;
   1144         }
   1145         state->set(FLUSHING);
   1146         return OK;
   1147     }();
   1148     switch (err) {
   1149         case ALREADY_EXISTS:
   1150             mCallback->onFlushCompleted();
   1151             return;
   1152         case OK:
   1153             break;
   1154         default:
   1155             mCallback->onError(err, ACTION_CODE_FATAL);
   1156             return;
   1157     }
   1158 
   1159     mChannel->stop();
   1160     (new AMessage(kWhatFlush, this))->post();
   1161 }
   1162 
   1163 void CCodec::flush() {
   1164     std::shared_ptr<Codec2Client::Component> comp;
   1165     auto checkFlushing = [this, &comp] {
   1166         Mutexed<State>::Locked state(mState);
   1167         if (state->get() != FLUSHING) {
   1168             return UNKNOWN_ERROR;
   1169         }
   1170         comp = state->comp;
   1171         return OK;
   1172     };
   1173     if (tryAndReportOnError(checkFlushing) != OK) {
   1174         return;
   1175     }
   1176 
   1177     std::list<std::unique_ptr<C2Work>> flushedWork;
   1178     c2_status_t err = comp->flush(C2Component::FLUSH_COMPONENT, &flushedWork);
   1179     if (err != C2_OK) {
   1180         // TODO: convert err into status_t
   1181         mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
   1182     }
   1183 
   1184     mChannel->flush(flushedWork);
   1185 
   1186     {
   1187         Mutexed<State>::Locked state(mState);
   1188         state->set(FLUSHED);
   1189     }
   1190     mCallback->onFlushCompleted();
   1191 }
   1192 
   1193 void CCodec::signalResume() {
   1194     auto setResuming = [this] {
   1195         Mutexed<State>::Locked state(mState);
   1196         if (state->get() != FLUSHED) {
   1197             return UNKNOWN_ERROR;
   1198         }
   1199         state->set(RESUMING);
   1200         return OK;
   1201     };
   1202     if (tryAndReportOnError(setResuming) != OK) {
   1203         return;
   1204     }
   1205 
   1206     (void)mChannel->start(nullptr, nullptr);
   1207 
   1208     {
   1209         Mutexed<State>::Locked state(mState);
   1210         if (state->get() != RESUMING) {
   1211             state.unlock();
   1212             mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
   1213             state.lock();
   1214             return;
   1215         }
   1216         state->set(RUNNING);
   1217     }
   1218 }
   1219 
   1220 void CCodec::signalSetParameters(const sp<AMessage> &params) {
   1221     sp<AMessage> msg = new AMessage(kWhatSetParameters, this);
   1222     msg->setMessage("params", params);
   1223     msg->post();
   1224 }
   1225 
   1226 void CCodec::setParameters(const sp<AMessage> &params) {
   1227     std::shared_ptr<Codec2Client::Component> comp;
   1228     auto checkState = [this, &comp] {
   1229         Mutexed<State>::Locked state(mState);
   1230         if (state->get() == RELEASED) {
   1231             return INVALID_OPERATION;
   1232         }
   1233         comp = state->comp;
   1234         return OK;
   1235     };
   1236     if (tryAndReportOnError(checkState) != OK) {
   1237         return;
   1238     }
   1239 
   1240     Mutexed<Config>::Locked config(mConfig);
   1241 
   1242     /**
   1243      * Handle input surface parameters
   1244      */
   1245     if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))
   1246             && (config->mDomain & Config::IS_ENCODER) && config->mInputSurface && config->mISConfig) {
   1247         (void)params->findInt64("time-offset-us", &config->mISConfig->mTimeOffsetUs);
   1248 
   1249         if (params->findInt64("skip-frames-before", &config->mISConfig->mStartAtUs)) {
   1250             config->mISConfig->mStopped = false;
   1251         } else if (params->findInt64("stop-time-us", &config->mISConfig->mStopAtUs)) {
   1252             config->mISConfig->mStopped = true;
   1253         }
   1254 
   1255         int32_t value;
   1256         if (params->findInt32("drop-input-frames", &value)) {
   1257             config->mISConfig->mSuspended = value;
   1258             config->mISConfig->mSuspendAtUs = -1;
   1259             (void)params->findInt64("drop-start-time-us", &config->mISConfig->mSuspendAtUs);
   1260         }
   1261 
   1262         (void)config->mInputSurface->configure(*config->mISConfig);
   1263         if (config->mISConfig->mStopped) {
   1264             config->mInputFormat->setInt64(
   1265                     "android._stop-time-offset-us", config->mISConfig->mInputDelayUs);
   1266         }
   1267     }
   1268 
   1269     std::vector<std::unique_ptr<C2Param>> configUpdate;
   1270     (void)config->getConfigUpdateFromSdkParams(comp, params, Config::PARAM, C2_MAY_BLOCK, &configUpdate);
   1271     if (property_get_bool("debug.stagefright.ccodec_delayed_params", false)) {
   1272         // mChannel->queueConfigUpdate(configUpdate);
   1273     } else {
   1274         (void)config->setParameters(comp, configUpdate, C2_MAY_BLOCK);
   1275     }
   1276 }
   1277 
   1278 void CCodec::signalEndOfInputStream() {
   1279     mCallback->onSignaledInputEOS(mChannel->signalEndOfInputStream());
   1280 }
   1281 
   1282 void CCodec::signalRequestIDRFrame() {
   1283     // TODO
   1284 }
   1285 
   1286 void CCodec::onWorkDone(std::list<std::unique_ptr<C2Work>> &workItems) {
   1287     Mutexed<std::list<std::unique_ptr<C2Work>>>::Locked queue(mWorkDoneQueue);
   1288     queue->splice(queue->end(), workItems);
   1289     (new AMessage(kWhatWorkDone, this))->post();
   1290 }
   1291 
   1292 void CCodec::onMessageReceived(const sp<AMessage> &msg) {
   1293     TimePoint now = std::chrono::steady_clock::now();
   1294     switch (msg->what()) {
   1295         case kWhatAllocate: {
   1296             // C2ComponentStore::createComponent() should return within 100ms.
   1297             setDeadline(now + 150ms, "allocate");
   1298             sp<RefBase> obj;
   1299             CHECK(msg->findObject("codecInfo", &obj));
   1300             allocate((MediaCodecInfo *)obj.get());
   1301             break;
   1302         }
   1303         case kWhatConfigure: {
   1304             // C2Component::commit_sm() should return within 5ms.
   1305             setDeadline(now + 50ms, "configure");
   1306             sp<AMessage> format;
   1307             CHECK(msg->findMessage("format", &format));
   1308             configure(format);
   1309             break;
   1310         }
   1311         case kWhatStart: {
   1312             // C2Component::start() should return within 500ms.
   1313             setDeadline(now + 550ms, "start");
   1314             start();
   1315             break;
   1316         }
   1317         case kWhatStop: {
   1318             // C2Component::stop() should return within 500ms.
   1319             setDeadline(now + 550ms, "stop");
   1320             stop();
   1321             break;
   1322         }
   1323         case kWhatFlush: {
   1324             // C2Component::flush_sm() should return within 5ms.
   1325             setDeadline(now + 50ms, "flush");
   1326             flush();
   1327             break;
   1328         }
   1329         case kWhatCreateInputSurface: {
   1330             // Surface operations may be briefly blocking.
   1331             setDeadline(now + 100ms, "createInputSurface");
   1332             createInputSurface();
   1333             break;
   1334         }
   1335         case kWhatSetInputSurface: {
   1336             // Surface operations may be briefly blocking.
   1337             setDeadline(now + 100ms, "setInputSurface");
   1338             sp<RefBase> obj;
   1339             CHECK(msg->findObject("surface", &obj));
   1340             sp<PersistentSurface> surface(static_cast<PersistentSurface *>(obj.get()));
   1341             setInputSurface(surface);
   1342             break;
   1343         }
   1344         case kWhatSetParameters: {
   1345             setDeadline(now + 50ms, "setParameters");
   1346             sp<AMessage> params;
   1347             CHECK(msg->findMessage("params", &params));
   1348             setParameters(params);
   1349             break;
   1350         }
   1351         case kWhatWorkDone: {
   1352             std::unique_ptr<C2Work> work;
   1353             {
   1354                 Mutexed<std::list<std::unique_ptr<C2Work>>>::Locked queue(mWorkDoneQueue);
   1355                 if (queue->empty()) {
   1356                     break;
   1357                 }
   1358                 work.swap(queue->front());
   1359                 queue->pop_front();
   1360                 if (!queue->empty()) {
   1361                     (new AMessage(kWhatWorkDone, this))->post();
   1362                 }
   1363             }
   1364 
   1365             // handle configuration changes in work done
   1366             Mutexed<Config>::Locked config(mConfig);
   1367             bool changed = false;
   1368             Config::Watcher<C2StreamInitDataInfo::output> initData =
   1369                 config->watch<C2StreamInitDataInfo::output>();
   1370             if (!work->worklets.empty()
   1371                     && (work->worklets.front()->output.flags
   1372                             & C2FrameData::FLAG_DISCARD_FRAME) == 0) {
   1373 
   1374                 // copy buffer info to config
   1375                 std::vector<std::unique_ptr<C2Param>> updates =
   1376                     std::move(work->worklets.front()->output.configUpdate);
   1377                 unsigned stream = 0;
   1378                 for (const std::shared_ptr<C2Buffer> &buf : work->worklets.front()->output.buffers) {
   1379                     for (const std::shared_ptr<const C2Info> &info : buf->info()) {
   1380                         // move all info into output-stream #0 domain
   1381                         updates.emplace_back(C2Param::CopyAsStream(*info, true /* output */, stream));
   1382                     }
   1383                     for (const C2ConstGraphicBlock &block : buf->data().graphicBlocks()) {
   1384                         // ALOGV("got output buffer with crop %u,%u+%u,%u and size %u,%u",
   1385                         //      block.crop().left, block.crop().top,
   1386                         //      block.crop().width, block.crop().height,
   1387                         //      block.width(), block.height());
   1388                         updates.emplace_back(new C2StreamCropRectInfo::output(stream, block.crop()));
   1389                         updates.emplace_back(new C2StreamPictureSizeInfo::output(
   1390                                 stream, block.width(), block.height()));
   1391                         break; // for now only do the first block
   1392                     }
   1393                     ++stream;
   1394                 }
   1395 
   1396                 changed = config->updateConfiguration(updates, config->mOutputDomain);
   1397 
   1398                 // copy standard infos to graphic buffers if not already present (otherwise, we
   1399                 // may overwrite the actual intermediate value with a final value)
   1400                 stream = 0;
   1401                 const static std::vector<C2Param::Index> stdGfxInfos = {
   1402                     C2StreamRotationInfo::output::PARAM_TYPE,
   1403                     C2StreamColorAspectsInfo::output::PARAM_TYPE,
   1404                     C2StreamHdrStaticInfo::output::PARAM_TYPE,
   1405                     C2StreamPixelAspectRatioInfo::output::PARAM_TYPE,
   1406                     C2StreamSurfaceScalingInfo::output::PARAM_TYPE
   1407                 };
   1408                 for (const std::shared_ptr<C2Buffer> &buf : work->worklets.front()->output.buffers) {
   1409                     if (buf->data().graphicBlocks().size()) {
   1410                         for (C2Param::Index ix : stdGfxInfos) {
   1411                             if (!buf->hasInfo(ix)) {
   1412                                 const C2Param *param =
   1413                                     config->getConfigParameterValue(ix.withStream(stream));
   1414                                 if (param) {
   1415                                     std::shared_ptr<C2Param> info(C2Param::Copy(*param));
   1416                                     buf->setInfo(std::static_pointer_cast<C2Info>(info));
   1417                                 }
   1418                             }
   1419                         }
   1420                     }
   1421                     ++stream;
   1422                 }
   1423             }
   1424             mChannel->onWorkDone(
   1425                     std::move(work), changed ? config->mOutputFormat : nullptr,
   1426                     initData.hasChanged() ? initData.update().get() : nullptr);
   1427             break;
   1428         }
   1429         default: {
   1430             ALOGE("unrecognized message");
   1431             break;
   1432         }
   1433     }
   1434     setDeadline(TimePoint::max(), "none");
   1435 }
   1436 
   1437 void CCodec::setDeadline(const TimePoint &newDeadline, const char *name) {
   1438     Mutexed<NamedTimePoint>::Locked deadline(mDeadline);
   1439     deadline->set(newDeadline, name);
   1440 }
   1441 
   1442 void CCodec::initiateReleaseIfStuck() {
   1443     std::string name;
   1444     {
   1445         Mutexed<NamedTimePoint>::Locked deadline(mDeadline);
   1446         if (deadline->get() >= std::chrono::steady_clock::now()) {
   1447             // We're not stuck.
   1448             return;
   1449         }
   1450         name = deadline->getName();
   1451     }
   1452 
   1453     ALOGW("previous call to %s exceeded timeout", name.c_str());
   1454     initiateRelease(false);
   1455     mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
   1456 }
   1457 
   1458 }  // namespace android
   1459 
   1460 extern "C" android::CodecBase *CreateCodec() {
   1461     return new android::CCodec;
   1462 }
   1463 
   1464