Home | History | Annotate | Download | only in stagefright 
      1 /*
      2  * Copyright 2012, The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *     http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #ifndef MEDIA_CODEC_H_
     18 
     19 #define MEDIA_CODEC_H_
     20 
     21 #include <memory>
     22 #include <vector>
     23 
     24 #include <gui/IGraphicBufferProducer.h>
     25 #include <media/hardware/CryptoAPI.h>
     26 #include <media/MediaCodecInfo.h>
     27 #include <media/MediaResource.h>
     28 #include <media/MediaAnalyticsItem.h>
     29 #include <media/stagefright/foundation/AHandler.h>
     30 #include <media/stagefright/FrameRenderTracker.h>
     31 #include <utils/Vector.h>
     32 
     33 namespace android {
     34 
     35 struct ABuffer;
     36 struct AMessage;
     37 struct AReplyToken;
     38 struct AString;
     39 class BufferChannelBase;
     40 struct CodecBase;
     41 class IBatteryStats;
     42 struct ICrypto;
     43 class MediaCodecBuffer;
     44 class IMemory;
     45 class IResourceManagerClient;
     46 class IResourceManagerService;
     47 struct PersistentSurface;
     48 class SoftwareRenderer;
     49 class Surface;
     50 namespace hardware {
     51 namespace cas {
     52 namespace native {
     53 namespace V1_0 {
     54 struct IDescrambler;
     55 }}}}
     56 using hardware::cas::native::V1_0::IDescrambler;
     57 
     58 struct MediaCodec : public AHandler {
     59     enum ConfigureFlags {
     60         CONFIGURE_FLAG_ENCODE   = 1,
     61     };
     62 
     63     enum BufferFlags {
     64         BUFFER_FLAG_SYNCFRAME   = 1,
     65         BUFFER_FLAG_CODECCONFIG = 2,
     66         BUFFER_FLAG_EOS         = 4,
     67     };
     68 
     69     enum {
     70         CB_INPUT_AVAILABLE = 1,
     71         CB_OUTPUT_AVAILABLE = 2,
     72         CB_ERROR = 3,
     73         CB_OUTPUT_FORMAT_CHANGED = 4,
     74         CB_RESOURCE_RECLAIMED = 5,
     75     };
     76 
     77     static const pid_t kNoPid = -1;
     78     static const uid_t kNoUid = -1;
     79 
     80     static sp<MediaCodec> CreateByType(
     81             const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err = NULL,
     82             pid_t pid = kNoPid, uid_t uid = kNoUid);
     83 
     84     static sp<MediaCodec> CreateByComponentName(
     85             const sp<ALooper> &looper, const AString &name, status_t *err = NULL,
     86             pid_t pid = kNoPid, uid_t uid = kNoUid);
     87 
     88     static sp<PersistentSurface> CreatePersistentInputSurface();
     89 
     90     status_t configure(
     91             const sp<AMessage> &format,
     92             const sp<Surface> &nativeWindow,
     93             const sp<ICrypto> &crypto,
     94             uint32_t flags);
     95 
     96     status_t configure(
     97             const sp<AMessage> &format,
     98             const sp<Surface> &nativeWindow,
     99             const sp<ICrypto> &crypto,
    100             const sp<IDescrambler> &descrambler,
    101             uint32_t flags);
    102 
    103     status_t releaseCrypto();
    104 
    105     status_t setCallback(const sp<AMessage> &callback);
    106 
    107     status_t setOnFrameRenderedNotification(const sp<AMessage> &notify);
    108 
    109     status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
    110 
    111     status_t setInputSurface(const sp<PersistentSurface> &surface);
    112 
    113     status_t start();
    114 
    115     // Returns to a state in which the component remains allocated but
    116     // unconfigured.
    117     status_t stop();
    118 
    119     // Resets the codec to the INITIALIZED state.  Can be called after an error
    120     // has occured to make the codec usable.
    121     status_t reset();
    122 
    123     // Client MUST call release before releasing final reference to this
    124     // object.
    125     status_t release();
    126 
    127     status_t flush();
    128 
    129     status_t queueInputBuffer(
    130             size_t index,
    131             size_t offset,
    132             size_t size,
    133             int64_t presentationTimeUs,
    134             uint32_t flags,
    135             AString *errorDetailMsg = NULL);
    136 
    137     status_t queueSecureInputBuffer(
    138             size_t index,
    139             size_t offset,
    140             const CryptoPlugin::SubSample *subSamples,
    141             size_t numSubSamples,
    142             const uint8_t key[16],
    143             const uint8_t iv[16],
    144             CryptoPlugin::Mode mode,
    145             const CryptoPlugin::Pattern &pattern,
    146             int64_t presentationTimeUs,
    147             uint32_t flags,
    148             AString *errorDetailMsg = NULL);
    149 
    150     status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll);
    151 
    152     status_t dequeueOutputBuffer(
    153             size_t *index,
    154             size_t *offset,
    155             size_t *size,
    156             int64_t *presentationTimeUs,
    157             uint32_t *flags,
    158             int64_t timeoutUs = 0ll);
    159 
    160     status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs);
    161     status_t renderOutputBufferAndRelease(size_t index);
    162     status_t releaseOutputBuffer(size_t index);
    163 
    164     status_t signalEndOfInputStream();
    165 
    166     status_t getOutputFormat(sp<AMessage> *format) const;
    167     status_t getInputFormat(sp<AMessage> *format) const;
    168 
    169     status_t getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
    170     status_t getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
    171 
    172     status_t getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
    173     status_t getOutputFormat(size_t index, sp<AMessage> *format);
    174     status_t getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
    175 
    176     status_t setSurface(const sp<Surface> &nativeWindow);
    177 
    178     status_t requestIDRFrame();
    179 
    180     // Notification will be posted once there "is something to do", i.e.
    181     // an input/output buffer has become available, a format change is
    182     // pending, an error is pending.
    183     void requestActivityNotification(const sp<AMessage> &notify);
    184 
    185     status_t getName(AString *componentName) const;
    186 
    187     status_t getMetrics(MediaAnalyticsItem * &reply);
    188 
    189     status_t setParameters(const sp<AMessage> &params);
    190 
    191     // Create a MediaCodec notification message from a list of rendered or dropped render infos
    192     // by adding rendered frame information to a base notification message. Returns the number
    193     // of frames that were rendered.
    194     static size_t CreateFramesRenderedMessage(
    195             const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg);
    196 
    197 protected:
    198     virtual ~MediaCodec();
    199     virtual void onMessageReceived(const sp<AMessage> &msg);
    200 
    201 private:
    202     // used by ResourceManagerClient
    203     status_t reclaim(bool force = false);
    204     friend struct ResourceManagerClient;
    205 
    206 private:
    207     enum State {
    208         UNINITIALIZED,
    209         INITIALIZING,
    210         INITIALIZED,
    211         CONFIGURING,
    212         CONFIGURED,
    213         STARTING,
    214         STARTED,
    215         FLUSHING,
    216         FLUSHED,
    217         STOPPING,
    218         RELEASING,
    219     };
    220 
    221     enum {
    222         kPortIndexInput         = 0,
    223         kPortIndexOutput        = 1,
    224     };
    225 
    226     enum {
    227         kWhatInit                           = 'init',
    228         kWhatConfigure                      = 'conf',
    229         kWhatSetSurface                     = 'sSur',
    230         kWhatCreateInputSurface             = 'cisf',
    231         kWhatSetInputSurface                = 'sisf',
    232         kWhatStart                          = 'strt',
    233         kWhatStop                           = 'stop',
    234         kWhatRelease                        = 'rele',
    235         kWhatDequeueInputBuffer             = 'deqI',
    236         kWhatQueueInputBuffer               = 'queI',
    237         kWhatDequeueOutputBuffer            = 'deqO',
    238         kWhatReleaseOutputBuffer            = 'relO',
    239         kWhatSignalEndOfInputStream         = 'eois',
    240         kWhatGetBuffers                     = 'getB',
    241         kWhatFlush                          = 'flus',
    242         kWhatGetOutputFormat                = 'getO',
    243         kWhatGetInputFormat                 = 'getI',
    244         kWhatDequeueInputTimedOut           = 'dITO',
    245         kWhatDequeueOutputTimedOut          = 'dOTO',
    246         kWhatCodecNotify                    = 'codc',
    247         kWhatRequestIDRFrame                = 'ridr',
    248         kWhatRequestActivityNotification    = 'racN',
    249         kWhatGetName                        = 'getN',
    250         kWhatSetParameters                  = 'setP',
    251         kWhatSetCallback                    = 'setC',
    252         kWhatSetNotification                = 'setN',
    253         kWhatDrmReleaseCrypto               = 'rDrm',
    254     };
    255 
    256     enum {
    257         kFlagUsesSoftwareRenderer       = 1,
    258         kFlagOutputFormatChanged        = 2,
    259         kFlagOutputBuffersChanged       = 4,
    260         kFlagStickyError                = 8,
    261         kFlagDequeueInputPending        = 16,
    262         kFlagDequeueOutputPending       = 32,
    263         kFlagIsSecure                   = 64,
    264         kFlagSawMediaServerDie          = 128,
    265         kFlagIsEncoder                  = 256,
    266         // 512 skipped
    267         kFlagIsAsync                    = 1024,
    268         kFlagIsComponentAllocated       = 2048,
    269         kFlagPushBlankBuffersOnShutdown = 4096,
    270     };
    271 
    272     struct BufferInfo {
    273         BufferInfo();
    274 
    275         sp<MediaCodecBuffer> mData;
    276         bool mOwnedByClient;
    277     };
    278 
    279     struct ResourceManagerServiceProxy : public IBinder::DeathRecipient {
    280         ResourceManagerServiceProxy(pid_t pid);
    281         ~ResourceManagerServiceProxy();
    282 
    283         void init();
    284 
    285         // implements DeathRecipient
    286         virtual void binderDied(const wp<IBinder>& /*who*/);
    287 
    288         void addResource(
    289                 int64_t clientId,
    290                 const sp<IResourceManagerClient> &client,
    291                 const Vector<MediaResource> &resources);
    292 
    293         void removeResource(int64_t clientId);
    294 
    295         bool reclaimResource(const Vector<MediaResource> &resources);
    296 
    297     private:
    298         Mutex mLock;
    299         sp<IResourceManagerService> mService;
    300         pid_t mPid;
    301     };
    302 
    303     State mState;
    304     uid_t mUid;
    305     bool mReleasedByResourceManager;
    306     sp<ALooper> mLooper;
    307     sp<ALooper> mCodecLooper;
    308     sp<CodecBase> mCodec;
    309     AString mComponentName;
    310     sp<AReplyToken> mReplyID;
    311     uint32_t mFlags;
    312     status_t mStickyError;
    313     sp<Surface> mSurface;
    314     SoftwareRenderer *mSoftRenderer;
    315 
    316     MediaAnalyticsItem *mAnalyticsItem;
    317     void initAnalyticsItem();
    318     void flushAnalyticsItem();
    319 
    320     sp<AMessage> mOutputFormat;
    321     sp<AMessage> mInputFormat;
    322     sp<AMessage> mCallback;
    323     sp<AMessage> mOnFrameRenderedNotification;
    324 
    325     sp<IResourceManagerClient> mResourceManagerClient;
    326     sp<ResourceManagerServiceProxy> mResourceManagerService;
    327 
    328     bool mBatteryStatNotified;
    329     bool mIsVideo;
    330     int32_t mVideoWidth;
    331     int32_t mVideoHeight;
    332     int32_t mRotationDegrees;
    333 
    334     // initial create parameters
    335     AString mInitName;
    336     bool mInitNameIsType;
    337     bool mInitIsEncoder;
    338 
    339     // configure parameter
    340     sp<AMessage> mConfigureMsg;
    341 
    342     // Used only to synchronize asynchronous getBufferAndFormat
    343     // across all the other (synchronous) buffer state change
    344     // operations, such as de/queueIn/OutputBuffer, start and
    345     // stop/flush/reset/release.
    346     Mutex mBufferLock;
    347 
    348     List<size_t> mAvailPortBuffers[2];
    349     std::vector<BufferInfo> mPortBuffers[2];
    350 
    351     int32_t mDequeueInputTimeoutGeneration;
    352     sp<AReplyToken> mDequeueInputReplyID;
    353 
    354     int32_t mDequeueOutputTimeoutGeneration;
    355     sp<AReplyToken> mDequeueOutputReplyID;
    356 
    357     sp<ICrypto> mCrypto;
    358 
    359     sp<IDescrambler> mDescrambler;
    360 
    361     List<sp<ABuffer> > mCSD;
    362 
    363     sp<AMessage> mActivityNotify;
    364 
    365     bool mHaveInputSurface;
    366     bool mHavePendingInputBuffers;
    367 
    368     std::shared_ptr<BufferChannelBase> mBufferChannel;
    369 
    370     MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid);
    371 
    372     static sp<CodecBase> GetCodecBase(const AString &name, bool nameIsType = false);
    373 
    374     static status_t PostAndAwaitResponse(
    375             const sp<AMessage> &msg, sp<AMessage> *response);
    376 
    377     void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err);
    378 
    379     status_t init(const AString &name, bool nameIsType, bool encoder);
    380 
    381     void setState(State newState);
    382     void returnBuffersToCodec(bool isReclaim = false);
    383     void returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim = false);
    384     size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg);
    385     status_t onQueueInputBuffer(const sp<AMessage> &msg);
    386     status_t onReleaseOutputBuffer(const sp<AMessage> &msg);
    387     ssize_t dequeuePortBuffer(int32_t portIndex);
    388 
    389     status_t getBufferAndFormat(
    390             size_t portIndex, size_t index,
    391             sp<MediaCodecBuffer> *buffer, sp<AMessage> *format);
    392 
    393     bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
    394     bool handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
    395     void cancelPendingDequeueOperations();
    396 
    397     void extractCSD(const sp<AMessage> &format);
    398     status_t queueCSDInputBuffer(size_t bufferIndex);
    399 
    400     status_t handleSetSurface(const sp<Surface> &surface);
    401     status_t connectToSurface(const sp<Surface> &surface);
    402     status_t disconnectFromSurface();
    403 
    404     bool hasCryptoOrDescrambler() {
    405         return mCrypto != NULL || mDescrambler != NULL;
    406     }
    407 
    408     void postActivityNotificationIfPossible();
    409 
    410     void onInputBufferAvailable();
    411     void onOutputBufferAvailable();
    412     void onError(status_t err, int32_t actionCode, const char *detail = NULL);
    413     void onOutputFormatChanged();
    414 
    415     status_t onSetParameters(const sp<AMessage> &params);
    416 
    417     status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer);
    418     void updateBatteryStat();
    419     bool isExecuting() const;
    420 
    421     uint64_t getGraphicBufferSize();
    422     void addResource(MediaResource::Type type, MediaResource::SubType subtype, uint64_t value);
    423 
    424     bool hasPendingBuffer(int portIndex);
    425     bool hasPendingBuffer();
    426 
    427     /* called to get the last codec error when the sticky flag is set.
    428      * if no such codec error is found, returns UNKNOWN_ERROR.
    429      */
    430     inline status_t getStickyError() const {
    431         return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR;
    432     }
    433 
    434     inline void setStickyError(status_t err) {
    435         mFlags |= kFlagStickyError;
    436         mStickyError = err;
    437     }
    438 
    439     void onReleaseCrypto(const sp<AMessage>& msg);
    440 
    441     DISALLOW_EVIL_CONSTRUCTORS(MediaCodec);
    442 };
    443 
    444 }  // namespace android
    445 
    446 #endif  // MEDIA_CODEC_H_
    447