Home | History | Annotate | Download | only in stagefright 
      1 /*
      2  * Copyright 2012, The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *     http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #ifndef MEDIA_CODEC_H_
     18 
     19 #define MEDIA_CODEC_H_
     20 
     21 #include <memory>
     22 #include <vector>
     23 
     24 #include <gui/IGraphicBufferProducer.h>
     25 #include <media/hardware/CryptoAPI.h>
     26 #include <media/MediaCodecInfo.h>
     27 #include <media/MediaResource.h>
     28 #include <media/MediaAnalyticsItem.h>
     29 #include <media/stagefright/foundation/AHandler.h>
     30 #include <media/stagefright/FrameRenderTracker.h>
     31 #include <utils/Vector.h>
     32 
     33 namespace android {
     34 
     35 struct ABuffer;
     36 struct AMessage;
     37 struct AReplyToken;
     38 struct AString;
     39 class BufferChannelBase;
     40 struct CodecBase;
     41 class IBatteryStats;
     42 struct ICrypto;
     43 class MediaCodecBuffer;
     44 class IMemory;
     45 class IResourceManagerClient;
     46 class IResourceManagerService;
     47 struct PersistentSurface;
     48 class SoftwareRenderer;
     49 class Surface;
     50 namespace hardware {
     51 namespace cas {
     52 namespace native {
     53 namespace V1_0 {
     54 struct IDescrambler;
     55 }}}}
     56 using hardware::cas::native::V1_0::IDescrambler;
     57 
     58 struct MediaCodec : public AHandler {
     59     enum ConfigureFlags {
     60         CONFIGURE_FLAG_ENCODE   = 1,
     61     };
     62 
     63     enum BufferFlags {
     64         BUFFER_FLAG_SYNCFRAME     = 1,
     65         BUFFER_FLAG_CODECCONFIG   = 2,
     66         BUFFER_FLAG_EOS           = 4,
     67         BUFFER_FLAG_PARTIAL_FRAME = 8,
     68         BUFFER_FLAG_MUXER_DATA    = 16,
     69     };
     70 
     71     enum {
     72         CB_INPUT_AVAILABLE = 1,
     73         CB_OUTPUT_AVAILABLE = 2,
     74         CB_ERROR = 3,
     75         CB_OUTPUT_FORMAT_CHANGED = 4,
     76         CB_RESOURCE_RECLAIMED = 5,
     77     };
     78 
     79     static const pid_t kNoPid = -1;
     80     static const uid_t kNoUid = -1;
     81 
     82     static sp<MediaCodec> CreateByType(
     83             const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err = NULL,
     84             pid_t pid = kNoPid, uid_t uid = kNoUid);
     85 
     86     static sp<MediaCodec> CreateByComponentName(
     87             const sp<ALooper> &looper, const AString &name, status_t *err = NULL,
     88             pid_t pid = kNoPid, uid_t uid = kNoUid);
     89 
     90     static sp<PersistentSurface> CreatePersistentInputSurface();
     91 
     92     status_t configure(
     93             const sp<AMessage> &format,
     94             const sp<Surface> &nativeWindow,
     95             const sp<ICrypto> &crypto,
     96             uint32_t flags);
     97 
     98     status_t configure(
     99             const sp<AMessage> &format,
    100             const sp<Surface> &nativeWindow,
    101             const sp<ICrypto> &crypto,
    102             const sp<IDescrambler> &descrambler,
    103             uint32_t flags);
    104 
    105     status_t releaseCrypto();
    106 
    107     status_t setCallback(const sp<AMessage> &callback);
    108 
    109     status_t setOnFrameRenderedNotification(const sp<AMessage> &notify);
    110 
    111     status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
    112 
    113     status_t setInputSurface(const sp<PersistentSurface> &surface);
    114 
    115     status_t start();
    116 
    117     // Returns to a state in which the component remains allocated but
    118     // unconfigured.
    119     status_t stop();
    120 
    121     // Resets the codec to the INITIALIZED state.  Can be called after an error
    122     // has occured to make the codec usable.
    123     status_t reset();
    124 
    125     // Client MUST call release before releasing final reference to this
    126     // object.
    127     status_t release();
    128 
    129     status_t flush();
    130 
    131     status_t queueInputBuffer(
    132             size_t index,
    133             size_t offset,
    134             size_t size,
    135             int64_t presentationTimeUs,
    136             uint32_t flags,
    137             AString *errorDetailMsg = NULL);
    138 
    139     status_t queueSecureInputBuffer(
    140             size_t index,
    141             size_t offset,
    142             const CryptoPlugin::SubSample *subSamples,
    143             size_t numSubSamples,
    144             const uint8_t key[16],
    145             const uint8_t iv[16],
    146             CryptoPlugin::Mode mode,
    147             const CryptoPlugin::Pattern &pattern,
    148             int64_t presentationTimeUs,
    149             uint32_t flags,
    150             AString *errorDetailMsg = NULL);
    151 
    152     status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll);
    153 
    154     status_t dequeueOutputBuffer(
    155             size_t *index,
    156             size_t *offset,
    157             size_t *size,
    158             int64_t *presentationTimeUs,
    159             uint32_t *flags,
    160             int64_t timeoutUs = 0ll);
    161 
    162     status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs);
    163     status_t renderOutputBufferAndRelease(size_t index);
    164     status_t releaseOutputBuffer(size_t index);
    165 
    166     status_t signalEndOfInputStream();
    167 
    168     status_t getOutputFormat(sp<AMessage> *format) const;
    169     status_t getInputFormat(sp<AMessage> *format) const;
    170 
    171     status_t getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
    172     status_t getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
    173 
    174     status_t getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
    175     status_t getOutputFormat(size_t index, sp<AMessage> *format);
    176     status_t getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
    177 
    178     status_t setSurface(const sp<Surface> &nativeWindow);
    179 
    180     status_t requestIDRFrame();
    181 
    182     // Notification will be posted once there "is something to do", i.e.
    183     // an input/output buffer has become available, a format change is
    184     // pending, an error is pending.
    185     void requestActivityNotification(const sp<AMessage> &notify);
    186 
    187     status_t getName(AString *componentName) const;
    188 
    189     status_t getCodecInfo(sp<MediaCodecInfo> *codecInfo) const;
    190 
    191     status_t getMetrics(MediaAnalyticsItem * &reply);
    192 
    193     status_t setParameters(const sp<AMessage> &params);
    194 
    195     // Create a MediaCodec notification message from a list of rendered or dropped render infos
    196     // by adding rendered frame information to a base notification message. Returns the number
    197     // of frames that were rendered.
    198     static size_t CreateFramesRenderedMessage(
    199             const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg);
    200 
    201 protected:
    202     virtual ~MediaCodec();
    203     virtual void onMessageReceived(const sp<AMessage> &msg);
    204 
    205 private:
    206     // used by ResourceManagerClient
    207     status_t reclaim(bool force = false);
    208     friend struct ResourceManagerClient;
    209 
    210 private:
    211     enum State {
    212         UNINITIALIZED,
    213         INITIALIZING,
    214         INITIALIZED,
    215         CONFIGURING,
    216         CONFIGURED,
    217         STARTING,
    218         STARTED,
    219         FLUSHING,
    220         FLUSHED,
    221         STOPPING,
    222         RELEASING,
    223     };
    224     std::string stateString(State state);
    225 
    226     enum {
    227         kPortIndexInput         = 0,
    228         kPortIndexOutput        = 1,
    229     };
    230 
    231     enum {
    232         kWhatInit                           = 'init',
    233         kWhatConfigure                      = 'conf',
    234         kWhatSetSurface                     = 'sSur',
    235         kWhatCreateInputSurface             = 'cisf',
    236         kWhatSetInputSurface                = 'sisf',
    237         kWhatStart                          = 'strt',
    238         kWhatStop                           = 'stop',
    239         kWhatRelease                        = 'rele',
    240         kWhatDequeueInputBuffer             = 'deqI',
    241         kWhatQueueInputBuffer               = 'queI',
    242         kWhatDequeueOutputBuffer            = 'deqO',
    243         kWhatReleaseOutputBuffer            = 'relO',
    244         kWhatSignalEndOfInputStream         = 'eois',
    245         kWhatGetBuffers                     = 'getB',
    246         kWhatFlush                          = 'flus',
    247         kWhatGetOutputFormat                = 'getO',
    248         kWhatGetInputFormat                 = 'getI',
    249         kWhatDequeueInputTimedOut           = 'dITO',
    250         kWhatDequeueOutputTimedOut          = 'dOTO',
    251         kWhatCodecNotify                    = 'codc',
    252         kWhatRequestIDRFrame                = 'ridr',
    253         kWhatRequestActivityNotification    = 'racN',
    254         kWhatGetName                        = 'getN',
    255         kWhatGetCodecInfo                   = 'gCoI',
    256         kWhatSetParameters                  = 'setP',
    257         kWhatSetCallback                    = 'setC',
    258         kWhatSetNotification                = 'setN',
    259         kWhatDrmReleaseCrypto               = 'rDrm',
    260     };
    261 
    262     enum {
    263         kFlagUsesSoftwareRenderer       = 1,
    264         kFlagOutputFormatChanged        = 2,
    265         kFlagOutputBuffersChanged       = 4,
    266         kFlagStickyError                = 8,
    267         kFlagDequeueInputPending        = 16,
    268         kFlagDequeueOutputPending       = 32,
    269         kFlagIsSecure                   = 64,
    270         kFlagSawMediaServerDie          = 128,
    271         kFlagIsEncoder                  = 256,
    272         // 512 skipped
    273         kFlagIsAsync                    = 1024,
    274         kFlagIsComponentAllocated       = 2048,
    275         kFlagPushBlankBuffersOnShutdown = 4096,
    276     };
    277 
    278     struct BufferInfo {
    279         BufferInfo();
    280 
    281         sp<MediaCodecBuffer> mData;
    282         bool mOwnedByClient;
    283     };
    284 
    285     struct ResourceManagerServiceProxy : public IBinder::DeathRecipient {
    286         ResourceManagerServiceProxy(pid_t pid);
    287         ~ResourceManagerServiceProxy();
    288 
    289         void init();
    290 
    291         // implements DeathRecipient
    292         virtual void binderDied(const wp<IBinder>& /*who*/);
    293 
    294         void addResource(
    295                 int64_t clientId,
    296                 const sp<IResourceManagerClient> &client,
    297                 const Vector<MediaResource> &resources);
    298 
    299         void removeResource(int64_t clientId);
    300 
    301         bool reclaimResource(const Vector<MediaResource> &resources);
    302 
    303     private:
    304         Mutex mLock;
    305         sp<IResourceManagerService> mService;
    306         pid_t mPid;
    307     };
    308 
    309     State mState;
    310     uid_t mUid;
    311     bool mReleasedByResourceManager;
    312     sp<ALooper> mLooper;
    313     sp<ALooper> mCodecLooper;
    314     sp<CodecBase> mCodec;
    315     AString mComponentName;
    316     sp<MediaCodecInfo> mCodecInfo;
    317     sp<AReplyToken> mReplyID;
    318     uint32_t mFlags;
    319     status_t mStickyError;
    320     sp<Surface> mSurface;
    321     SoftwareRenderer *mSoftRenderer;
    322 
    323     MediaAnalyticsItem *mAnalyticsItem;
    324     void initAnalyticsItem();
    325     void updateAnalyticsItem();
    326     void flushAnalyticsItem();
    327     void updateEphemeralAnalytics(MediaAnalyticsItem *item);
    328 
    329     sp<AMessage> mOutputFormat;
    330     sp<AMessage> mInputFormat;
    331     sp<AMessage> mCallback;
    332     sp<AMessage> mOnFrameRenderedNotification;
    333 
    334     sp<IResourceManagerClient> mResourceManagerClient;
    335     sp<ResourceManagerServiceProxy> mResourceManagerService;
    336 
    337     bool mBatteryStatNotified;
    338     bool mIsVideo;
    339     int32_t mVideoWidth;
    340     int32_t mVideoHeight;
    341     int32_t mRotationDegrees;
    342 
    343     // initial create parameters
    344     AString mInitName;
    345 
    346     // configure parameter
    347     sp<AMessage> mConfigureMsg;
    348 
    349     // Used only to synchronize asynchronous getBufferAndFormat
    350     // across all the other (synchronous) buffer state change
    351     // operations, such as de/queueIn/OutputBuffer, start and
    352     // stop/flush/reset/release.
    353     Mutex mBufferLock;
    354 
    355     List<size_t> mAvailPortBuffers[2];
    356     std::vector<BufferInfo> mPortBuffers[2];
    357 
    358     int32_t mDequeueInputTimeoutGeneration;
    359     sp<AReplyToken> mDequeueInputReplyID;
    360 
    361     int32_t mDequeueOutputTimeoutGeneration;
    362     sp<AReplyToken> mDequeueOutputReplyID;
    363 
    364     sp<ICrypto> mCrypto;
    365 
    366     sp<IDescrambler> mDescrambler;
    367 
    368     List<sp<ABuffer> > mCSD;
    369 
    370     sp<AMessage> mActivityNotify;
    371 
    372     bool mHaveInputSurface;
    373     bool mHavePendingInputBuffers;
    374     bool mCpuBoostRequested;
    375 
    376     std::shared_ptr<BufferChannelBase> mBufferChannel;
    377 
    378     MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid);
    379 
    380     static sp<CodecBase> GetCodecBase(const AString &name);
    381 
    382     static status_t PostAndAwaitResponse(
    383             const sp<AMessage> &msg, sp<AMessage> *response);
    384 
    385     void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err);
    386 
    387     status_t init(const AString &name);
    388 
    389     void setState(State newState);
    390     void returnBuffersToCodec(bool isReclaim = false);
    391     void returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim = false);
    392     size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg);
    393     status_t onQueueInputBuffer(const sp<AMessage> &msg);
    394     status_t onReleaseOutputBuffer(const sp<AMessage> &msg);
    395     ssize_t dequeuePortBuffer(int32_t portIndex);
    396 
    397     status_t getBufferAndFormat(
    398             size_t portIndex, size_t index,
    399             sp<MediaCodecBuffer> *buffer, sp<AMessage> *format);
    400 
    401     bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
    402     bool handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
    403     void cancelPendingDequeueOperations();
    404 
    405     void extractCSD(const sp<AMessage> &format);
    406     status_t queueCSDInputBuffer(size_t bufferIndex);
    407 
    408     status_t handleSetSurface(const sp<Surface> &surface);
    409     status_t connectToSurface(const sp<Surface> &surface);
    410     status_t disconnectFromSurface();
    411 
    412     bool hasCryptoOrDescrambler() {
    413         return mCrypto != NULL || mDescrambler != NULL;
    414     }
    415 
    416     void postActivityNotificationIfPossible();
    417 
    418     void onInputBufferAvailable();
    419     void onOutputBufferAvailable();
    420     void onError(status_t err, int32_t actionCode, const char *detail = NULL);
    421     void onOutputFormatChanged();
    422 
    423     status_t onSetParameters(const sp<AMessage> &params);
    424 
    425     status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer);
    426     void updateBatteryStat();
    427     bool isExecuting() const;
    428 
    429     uint64_t getGraphicBufferSize();
    430     void addResource(MediaResource::Type type, MediaResource::SubType subtype, uint64_t value);
    431     void requestCpuBoostIfNeeded();
    432 
    433     bool hasPendingBuffer(int portIndex);
    434     bool hasPendingBuffer();
    435 
    436     /* called to get the last codec error when the sticky flag is set.
    437      * if no such codec error is found, returns UNKNOWN_ERROR.
    438      */
    439     inline status_t getStickyError() const {
    440         return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR;
    441     }
    442 
    443     inline void setStickyError(status_t err) {
    444         mFlags |= kFlagStickyError;
    445         mStickyError = err;
    446     }
    447 
    448     void onReleaseCrypto(const sp<AMessage>& msg);
    449 
    450     // managing time-of-flight aka latency
    451     typedef struct {
    452             int64_t presentationUs;
    453             int64_t startedNs;
    454     } BufferFlightTiming_t;
    455     std::deque<BufferFlightTiming_t> mBuffersInFlight;
    456     Mutex mLatencyLock;
    457     int64_t mLatencyUnknown;    // buffers for which we couldn't calculate latency
    458 
    459     void statsBufferSent(int64_t presentationUs);
    460     void statsBufferReceived(int64_t presentationUs);
    461 
    462     enum {
    463         // the default shape of our latency histogram buckets
    464         // XXX: should these be configurable in some way?
    465         kLatencyHistBuckets = 20,
    466         kLatencyHistWidth = 2000,
    467         kLatencyHistFloor = 2000,
    468 
    469         // how many samples are in the 'recent latency' histogram
    470         // 300 frames = 5 sec @ 60fps or ~12 sec @ 24fps
    471         kRecentLatencyFrames = 300,
    472 
    473         // how we initialize mRecentSamples
    474         kRecentSampleInvalid = -1,
    475     };
    476 
    477     int64_t mRecentSamples[kRecentLatencyFrames];
    478     int mRecentHead;
    479     Mutex mRecentLock;
    480 
    481     class Histogram {
    482       public:
    483         Histogram() : mFloor(0), mWidth(0), mBelow(0), mAbove(0),
    484                       mMin(INT64_MAX), mMax(INT64_MIN), mSum(0), mCount(0),
    485                       mBucketCount(0), mBuckets(NULL) {};
    486         ~Histogram() { clear(); };
    487         void clear() { if (mBuckets != NULL) free(mBuckets); mBuckets = NULL; };
    488         bool setup(int nbuckets, int64_t width, int64_t floor = 0);
    489         void insert(int64_t sample);
    490         int64_t getMin() const { return mMin; }
    491         int64_t getMax() const { return mMax; }
    492         int64_t getCount() const { return mCount; }
    493         int64_t getSum() const { return mSum; }
    494         int64_t getAvg() const { return mSum / (mCount == 0 ? 1 : mCount); }
    495         std::string emit();
    496       private:
    497         int64_t mFloor, mCeiling, mWidth;
    498         int64_t mBelow, mAbove;
    499         int64_t mMin, mMax, mSum, mCount;
    500 
    501         int mBucketCount;
    502         int64_t *mBuckets;
    503     };
    504 
    505     Histogram mLatencyHist;
    506 
    507     DISALLOW_EVIL_CONSTRUCTORS(MediaCodec);
    508 };
    509 
    510 }  // namespace android
    511 
    512 #endif  // MEDIA_CODEC_H_
    513