1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef A_CODEC_H_ 18 19 #define A_CODEC_H_ 20 21 #include <stdint.h> 22 #include <android/native_window.h> 23 #include <media/hardware/MetadataBufferType.h> 24 #include <media/IOMX.h> 25 #include <media/stagefright/foundation/AHierarchicalStateMachine.h> 26 #include <media/stagefright/CodecBase.h> 27 #include <media/stagefright/FrameRenderTracker.h> 28 #include <media/stagefright/MediaDefs.h> 29 #include <media/stagefright/SkipCutBuffer.h> 30 #include <utils/NativeHandle.h> 31 #include <OMX_Audio.h> 32 33 #define TRACK_BUFFER_TIMING 0 34 35 namespace android { 36 37 struct ABuffer; 38 struct MemoryDealer; 39 struct DescribeColorFormat2Params; 40 struct DataConverter; 41 42 struct ACodec : public AHierarchicalStateMachine, public CodecBase { 43 ACodec(); 44 45 virtual void setNotificationMessage(const sp<AMessage> &msg); 46 47 void initiateSetup(const sp<AMessage> &msg); 48 49 virtual void initiateAllocateComponent(const sp<AMessage> &msg); 50 virtual void initiateConfigureComponent(const sp<AMessage> &msg); 51 virtual void initiateCreateInputSurface(); 52 virtual void initiateSetInputSurface(const sp<PersistentSurface> &surface); 53 virtual void initiateStart(); 54 virtual void initiateShutdown(bool keepComponentAllocated = false); 55 56 virtual status_t queryCapabilities( 57 const AString &name, const AString &mime, bool isEncoder, 58 sp<MediaCodecInfo::Capabilities> *caps); 59 60 virtual status_t setSurface(const sp<Surface> &surface); 61 62 virtual void signalFlush(); 63 virtual void signalResume(); 64 65 virtual void signalSetParameters(const sp<AMessage> &msg); 66 virtual void signalEndOfInputStream(); 67 virtual void signalRequestIDRFrame(); 68 69 // AHierarchicalStateMachine implements the message handling 70 virtual void onMessageReceived(const sp<AMessage> &msg) { 71 handleMessage(msg); 72 } 73 74 struct PortDescription : public CodecBase::PortDescription { 75 size_t countBuffers(); 76 IOMX::buffer_id bufferIDAt(size_t index) const; 77 sp<ABuffer> bufferAt(size_t index) const; 78 sp<NativeHandle> handleAt(size_t index) const; 79 sp<RefBase> memRefAt(size_t index) const; 80 81 private: 82 friend struct ACodec; 83 84 Vector<IOMX::buffer_id> mBufferIDs; 85 Vector<sp<ABuffer> > mBuffers; 86 Vector<sp<NativeHandle> > mHandles; 87 Vector<sp<RefBase> > mMemRefs; 88 89 PortDescription(); 90 void addBuffer( 91 IOMX::buffer_id id, const sp<ABuffer> &buffer, 92 const sp<NativeHandle> &handle, const sp<RefBase> &memRef); 93 94 DISALLOW_EVIL_CONSTRUCTORS(PortDescription); 95 }; 96 97 static bool isFlexibleColorFormat( 98 const sp<IOMX> &omx, IOMX::node_id node, 99 uint32_t colorFormat, bool usingNativeBuffers, OMX_U32 *flexibleEquivalent); 100 101 // Returns 0 if configuration is not supported. NOTE: this is treated by 102 // some OMX components as auto level, and by others as invalid level. 103 static int /* OMX_VIDEO_AVCLEVELTYPE */ getAVCLevelFor( 104 int width, int height, int rate, int bitrate, 105 OMX_VIDEO_AVCPROFILETYPE profile = OMX_VIDEO_AVCProfileBaseline); 106 107 // Quirk still supported, even though deprecated 108 enum Quirks { 109 kRequiresAllocateBufferOnInputPorts = 1, 110 kRequiresAllocateBufferOnOutputPorts = 2, 111 }; 112 113 static status_t getOMXChannelMapping(size_t numChannels, OMX_AUDIO_CHANNELTYPE map[]); 114 115 protected: 116 virtual ~ACodec(); 117 118 private: 119 struct BaseState; 120 struct UninitializedState; 121 struct LoadedState; 122 struct LoadedToIdleState; 123 struct IdleToExecutingState; 124 struct ExecutingState; 125 struct OutputPortSettingsChangedState; 126 struct ExecutingToIdleState; 127 struct IdleToLoadedState; 128 struct FlushingState; 129 struct DeathNotifier; 130 131 enum { 132 kWhatSetup = 'setu', 133 kWhatOMXMessage = 'omx ', 134 // same as kWhatOMXMessage - but only used with 135 // handleMessage during OMX message-list handling 136 kWhatOMXMessageItem = 'omxI', 137 kWhatOMXMessageList = 'omxL', 138 kWhatInputBufferFilled = 'inpF', 139 kWhatOutputBufferDrained = 'outD', 140 kWhatShutdown = 'shut', 141 kWhatFlush = 'flus', 142 kWhatResume = 'resm', 143 kWhatDrainDeferredMessages = 'drai', 144 kWhatAllocateComponent = 'allo', 145 kWhatConfigureComponent = 'conf', 146 kWhatSetSurface = 'setS', 147 kWhatCreateInputSurface = 'cisf', 148 kWhatSetInputSurface = 'sisf', 149 kWhatSignalEndOfInputStream = 'eois', 150 kWhatStart = 'star', 151 kWhatRequestIDRFrame = 'ridr', 152 kWhatSetParameters = 'setP', 153 kWhatSubmitOutputMetadataBufferIfEOS = 'subm', 154 kWhatOMXDied = 'OMXd', 155 kWhatReleaseCodecInstance = 'relC', 156 }; 157 158 enum { 159 kPortIndexInput = 0, 160 kPortIndexOutput = 1 161 }; 162 163 enum { 164 kFlagIsSecure = 1, 165 kFlagPushBlankBuffersToNativeWindowOnShutdown = 2, 166 kFlagIsGrallocUsageProtected = 4, 167 }; 168 169 enum { 170 kVideoGrallocUsage = (GRALLOC_USAGE_HW_TEXTURE 171 | GRALLOC_USAGE_HW_COMPOSER 172 | GRALLOC_USAGE_EXTERNAL_DISP), 173 }; 174 175 struct BufferInfo { 176 enum Status { 177 OWNED_BY_US, 178 OWNED_BY_COMPONENT, 179 OWNED_BY_UPSTREAM, 180 OWNED_BY_DOWNSTREAM, 181 OWNED_BY_NATIVE_WINDOW, 182 UNRECOGNIZED, // not a tracked buffer 183 }; 184 185 static inline Status getSafeStatus(BufferInfo *info) { 186 return info == NULL ? UNRECOGNIZED : info->mStatus; 187 } 188 189 IOMX::buffer_id mBufferID; 190 Status mStatus; 191 unsigned mDequeuedAt; 192 193 sp<ABuffer> mData; // the client's buffer; if not using data conversion, this is the 194 // codec buffer; otherwise, it is allocated separately 195 sp<RefBase> mMemRef; // and a reference to the IMemory, so it does not go away 196 sp<ABuffer> mCodecData; // the codec's buffer 197 sp<RefBase> mCodecRef; // and a reference to the IMemory 198 sp<GraphicBuffer> mGraphicBuffer; 199 sp<NativeHandle> mNativeHandle; 200 int mFenceFd; 201 FrameRenderTracker::Info *mRenderInfo; 202 203 // The following field and 4 methods are used for debugging only 204 bool mIsReadFence; 205 // Store |fenceFd| and set read/write flag. Log error, if there is already a fence stored. 206 void setReadFence(int fenceFd, const char *dbg); 207 void setWriteFence(int fenceFd, const char *dbg); 208 // Log error, if the current fence is not a read/write fence. 209 void checkReadFence(const char *dbg); 210 void checkWriteFence(const char *dbg); 211 }; 212 213 static const char *_asString(BufferInfo::Status s); 214 void dumpBuffers(OMX_U32 portIndex); 215 216 // If |fd| is non-negative, waits for fence with |fd| and logs an error if it fails. Returns 217 // the error code or OK on success. If |fd| is negative, it returns OK 218 status_t waitForFence(int fd, const char *dbg); 219 220 #if TRACK_BUFFER_TIMING 221 struct BufferStats { 222 int64_t mEmptyBufferTimeUs; 223 int64_t mFillBufferDoneTimeUs; 224 }; 225 226 KeyedVector<int64_t, BufferStats> mBufferStats; 227 #endif 228 229 sp<AMessage> mNotify; 230 231 sp<UninitializedState> mUninitializedState; 232 sp<LoadedState> mLoadedState; 233 sp<LoadedToIdleState> mLoadedToIdleState; 234 sp<IdleToExecutingState> mIdleToExecutingState; 235 sp<ExecutingState> mExecutingState; 236 sp<OutputPortSettingsChangedState> mOutputPortSettingsChangedState; 237 sp<ExecutingToIdleState> mExecutingToIdleState; 238 sp<IdleToLoadedState> mIdleToLoadedState; 239 sp<FlushingState> mFlushingState; 240 sp<SkipCutBuffer> mSkipCutBuffer; 241 242 AString mComponentName; 243 uint32_t mFlags; 244 uint32_t mQuirks; 245 sp<IOMX> mOMX; 246 sp<IBinder> mNodeBinder; 247 IOMX::node_id mNode; 248 sp<MemoryDealer> mDealer[2]; 249 250 bool mUsingNativeWindow; 251 sp<ANativeWindow> mNativeWindow; 252 int mNativeWindowUsageBits; 253 android_native_rect_t mLastNativeWindowCrop; 254 int32_t mLastNativeWindowDataSpace; 255 sp<AMessage> mConfigFormat; 256 sp<AMessage> mInputFormat; 257 sp<AMessage> mOutputFormat; 258 259 // Initial output format + configuration params that is reused as the base for all subsequent 260 // format updates. This will equal to mOutputFormat until the first actual frame is received. 261 sp<AMessage> mBaseOutputFormat; 262 263 FrameRenderTracker mRenderTracker; // render information for buffers rendered by ACodec 264 Vector<BufferInfo> mBuffers[2]; 265 bool mPortEOS[2]; 266 status_t mInputEOSResult; 267 268 List<sp<AMessage> > mDeferredQueue; 269 270 sp<AMessage> mLastOutputFormat; 271 bool mIsVideo; 272 bool mIsEncoder; 273 bool mFatalError; 274 bool mShutdownInProgress; 275 bool mExplicitShutdown; 276 bool mIsLegacyVP9Decoder; 277 278 // If "mKeepComponentAllocated" we only transition back to Loaded state 279 // and do not release the component instance. 280 bool mKeepComponentAllocated; 281 282 int32_t mEncoderDelay; 283 int32_t mEncoderPadding; 284 int32_t mRotationDegrees; 285 286 bool mChannelMaskPresent; 287 int32_t mChannelMask; 288 unsigned mDequeueCounter; 289 MetadataBufferType mInputMetadataType; 290 MetadataBufferType mOutputMetadataType; 291 bool mLegacyAdaptiveExperiment; 292 int32_t mMetadataBuffersToSubmit; 293 size_t mNumUndequeuedBuffers; 294 sp<DataConverter> mConverter[2]; 295 296 int64_t mRepeatFrameDelayUs; 297 int64_t mMaxPtsGapUs; 298 float mMaxFps; 299 300 int64_t mTimePerFrameUs; 301 int64_t mTimePerCaptureUs; 302 303 bool mCreateInputBuffersSuspended; 304 305 bool mTunneled; 306 307 OMX_INDEXTYPE mDescribeColorAspectsIndex; 308 OMX_INDEXTYPE mDescribeHDRStaticInfoIndex; 309 310 status_t setCyclicIntraMacroblockRefresh(const sp<AMessage> &msg, int32_t mode); 311 status_t allocateBuffersOnPort(OMX_U32 portIndex); 312 status_t freeBuffersOnPort(OMX_U32 portIndex); 313 status_t freeBuffer(OMX_U32 portIndex, size_t i); 314 315 status_t handleSetSurface(const sp<Surface> &surface); 316 status_t setupNativeWindowSizeFormatAndUsage( 317 ANativeWindow *nativeWindow /* nonnull */, int *finalUsage /* nonnull */, 318 bool reconnect); 319 320 status_t configureOutputBuffersFromNativeWindow( 321 OMX_U32 *nBufferCount, OMX_U32 *nBufferSize, 322 OMX_U32 *nMinUndequeuedBuffers, bool preregister); 323 status_t allocateOutputMetadataBuffers(); 324 status_t submitOutputMetadataBuffer(); 325 void signalSubmitOutputMetadataBufferIfEOS_workaround(); 326 status_t allocateOutputBuffersFromNativeWindow(); 327 status_t cancelBufferToNativeWindow(BufferInfo *info); 328 status_t freeOutputBuffersNotOwnedByComponent(); 329 BufferInfo *dequeueBufferFromNativeWindow(); 330 331 inline bool storingMetadataInDecodedBuffers() { 332 return mOutputMetadataType >= 0 && !mIsEncoder; 333 } 334 335 inline bool usingMetadataOnEncoderOutput() { 336 return mOutputMetadataType >= 0 && mIsEncoder; 337 } 338 339 BufferInfo *findBufferByID( 340 uint32_t portIndex, IOMX::buffer_id bufferID, 341 ssize_t *index = NULL); 342 343 status_t setComponentRole(bool isEncoder, const char *mime); 344 static const char *getComponentRole(bool isEncoder, const char *mime); 345 static status_t setComponentRole( 346 const sp<IOMX> &omx, IOMX::node_id node, const char *role); 347 348 status_t configureCodec(const char *mime, const sp<AMessage> &msg); 349 350 status_t configureTunneledVideoPlayback(int32_t audioHwSync, 351 const sp<ANativeWindow> &nativeWindow); 352 353 status_t setVideoPortFormatType( 354 OMX_U32 portIndex, 355 OMX_VIDEO_CODINGTYPE compressionFormat, 356 OMX_COLOR_FORMATTYPE colorFormat, 357 bool usingNativeBuffers = false); 358 359 status_t setSupportedOutputFormat(bool getLegacyFlexibleFormat); 360 361 status_t setupVideoDecoder( 362 const char *mime, const sp<AMessage> &msg, bool usingNativeBuffers, bool haveSwRenderer, 363 sp<AMessage> &outputformat); 364 365 status_t setupVideoEncoder( 366 const char *mime, const sp<AMessage> &msg, 367 sp<AMessage> &outputformat, sp<AMessage> &inputformat); 368 369 status_t setVideoFormatOnPort( 370 OMX_U32 portIndex, 371 int32_t width, int32_t height, 372 OMX_VIDEO_CODINGTYPE compressionFormat, float frameRate = -1.0); 373 374 // gets index or sets it to 0 on error. Returns error from codec. 375 status_t initDescribeColorAspectsIndex(); 376 377 // sets |params|. If |readBack| is true, it re-gets them afterwards if set succeeded. 378 // returns the codec error. 379 status_t setCodecColorAspects(DescribeColorAspectsParams ¶ms, bool readBack = false); 380 381 // gets |params|; returns the codec error. |param| should not change on error. 382 status_t getCodecColorAspects(DescribeColorAspectsParams ¶ms); 383 384 // gets dataspace guidance from codec and platform. |params| should be set up with the color 385 // aspects to use. If |tryCodec| is true, the codec is queried first. If it succeeds, we 386 // return OK. Otherwise, we fall back to the platform guidance and return the codec error; 387 // though, we return OK if the codec failed with UNSUPPORTED, as codec guidance is optional. 388 status_t getDataSpace( 389 DescribeColorAspectsParams ¶ms, android_dataspace *dataSpace /* nonnull */, 390 bool tryCodec); 391 392 // sets color aspects for the encoder for certain |width/height| based on |configFormat|, and 393 // set resulting color config into |outputFormat|. If |usingNativeWindow| is true, we use 394 // video defaults if config is unspecified. Returns error from the codec. 395 status_t setColorAspectsForVideoDecoder( 396 int32_t width, int32_t height, bool usingNativeWindow, 397 const sp<AMessage> &configFormat, sp<AMessage> &outputFormat); 398 399 // gets color aspects for the encoder for certain |width/height| based on |configFormat|, and 400 // set resulting color config into |outputFormat|. If |dataSpace| is non-null, it requests 401 // dataspace guidance from the codec and platform and sets it into |dataSpace|. Returns the 402 // error from the codec. 403 status_t getColorAspectsAndDataSpaceForVideoDecoder( 404 int32_t width, int32_t height, const sp<AMessage> &configFormat, 405 sp<AMessage> &outputFormat, android_dataspace *dataSpace); 406 407 // sets color aspects for the video encoder assuming bytebuffer mode for certain |configFormat| 408 // and sets resulting color config into |outputFormat|. For mediarecorder, also set dataspace 409 // into |inputFormat|. Returns the error from the codec. 410 status_t setColorAspectsForVideoEncoder( 411 const sp<AMessage> &configFormat, 412 sp<AMessage> &outputFormat, sp<AMessage> &inputFormat); 413 414 // sets color aspects for the video encoder in surface mode. This basically sets the default 415 // video values for unspecified aspects and sets the dataspace to use in the input format. 416 // Also sets the dataspace into |dataSpace|. 417 // Returns any codec errors during this configuration, except for optional steps. 418 status_t setInitialColorAspectsForVideoEncoderSurfaceAndGetDataSpace( 419 android_dataspace *dataSpace /* nonnull */); 420 421 // gets color aspects for the video encoder input port and sets them into the |format|. 422 // Returns any codec errors. 423 status_t getInputColorAspectsForVideoEncoder(sp<AMessage> &format); 424 425 // updates the encoder output format with |aspects| defaulting to |dataSpace| for 426 // unspecified values. 427 void onDataSpaceChanged(android_dataspace dataSpace, const ColorAspects &aspects); 428 429 // gets index or sets it to 0 on error. Returns error from codec. 430 status_t initDescribeHDRStaticInfoIndex(); 431 432 // sets HDR static metadata for the video encoder/decoder based on |configFormat|, and 433 // sets resulting HDRStaticInfo config into |outputFormat|. Returns error from the codec. 434 status_t setHDRStaticInfoForVideoCodec( 435 OMX_U32 portIndex, const sp<AMessage> &configFormat, sp<AMessage> &outputFormat); 436 437 // sets |params|. Returns the codec error. 438 status_t setHDRStaticInfo(const DescribeHDRStaticInfoParams ¶ms); 439 440 // gets |params|. Returns the codec error. 441 status_t getHDRStaticInfo(DescribeHDRStaticInfoParams ¶ms); 442 443 // gets HDR static information for the video encoder/decoder port and sets them into |format|. 444 status_t getHDRStaticInfoForVideoCodec(OMX_U32 portIndex, sp<AMessage> &format); 445 446 typedef struct drcParams { 447 int32_t drcCut; 448 int32_t drcBoost; 449 int32_t heavyCompression; 450 int32_t targetRefLevel; 451 int32_t encodedTargetLevel; 452 } drcParams_t; 453 454 status_t setupAACCodec( 455 bool encoder, 456 int32_t numChannels, int32_t sampleRate, int32_t bitRate, 457 int32_t aacProfile, bool isADTS, int32_t sbrMode, 458 int32_t maxOutputChannelCount, const drcParams_t& drc, 459 int32_t pcmLimiterEnable); 460 461 status_t setupAC3Codec(bool encoder, int32_t numChannels, int32_t sampleRate); 462 463 status_t setupEAC3Codec(bool encoder, int32_t numChannels, int32_t sampleRate); 464 465 status_t selectAudioPortFormat( 466 OMX_U32 portIndex, OMX_AUDIO_CODINGTYPE desiredFormat); 467 468 status_t setupAMRCodec(bool encoder, bool isWAMR, int32_t bitRate); 469 status_t setupG711Codec(bool encoder, int32_t sampleRate, int32_t numChannels); 470 471 status_t setupFlacCodec( 472 bool encoder, int32_t numChannels, int32_t sampleRate, int32_t compressionLevel); 473 474 status_t setupRawAudioFormat( 475 OMX_U32 portIndex, int32_t sampleRate, int32_t numChannels, 476 AudioEncoding encoding = kAudioEncodingPcm16bit); 477 478 status_t setPriority(int32_t priority); 479 status_t setOperatingRate(float rateFloat, bool isVideo); 480 status_t getIntraRefreshPeriod(uint32_t *intraRefreshPeriod); 481 status_t setIntraRefreshPeriod(uint32_t intraRefreshPeriod, bool inConfigure); 482 483 status_t setMinBufferSize(OMX_U32 portIndex, size_t size); 484 485 status_t setupMPEG4EncoderParameters(const sp<AMessage> &msg); 486 status_t setupH263EncoderParameters(const sp<AMessage> &msg); 487 status_t setupAVCEncoderParameters(const sp<AMessage> &msg); 488 status_t setupHEVCEncoderParameters(const sp<AMessage> &msg); 489 status_t setupVPXEncoderParameters(const sp<AMessage> &msg); 490 491 status_t verifySupportForProfileAndLevel(int32_t profile, int32_t level); 492 493 status_t configureBitrate( 494 int32_t bitrate, OMX_VIDEO_CONTROLRATETYPE bitrateMode); 495 496 status_t setupErrorCorrectionParameters(); 497 498 status_t initNativeWindow(); 499 500 // Returns true iff all buffers on the given port have status 501 // OWNED_BY_US or OWNED_BY_NATIVE_WINDOW. 502 bool allYourBuffersAreBelongToUs(OMX_U32 portIndex); 503 504 bool allYourBuffersAreBelongToUs(); 505 506 void waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs(); 507 508 size_t countBuffersOwnedByComponent(OMX_U32 portIndex) const; 509 size_t countBuffersOwnedByNativeWindow() const; 510 511 void deferMessage(const sp<AMessage> &msg); 512 void processDeferredMessages(); 513 514 void onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano); 515 // called when we have dequeued a buffer |buf| from the native window to track render info. 516 // |fenceFd| is the dequeue fence, and |info| points to the buffer info where this buffer is 517 // stored. 518 void updateRenderInfoForDequeuedBuffer( 519 ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info); 520 521 // Checks to see if any frames have rendered up until |until|, and to notify client 522 // (MediaCodec) of rendered frames up-until the frame pointed to by |until| or the first 523 // unrendered frame. These frames are removed from the render queue. 524 // If |dropIncomplete| is true, unrendered frames up-until |until| will be dropped from the 525 // queue, allowing all rendered framed up till then to be notified of. 526 // (This will effectively clear the render queue up-until (and including) |until|.) 527 // If |until| is NULL, or is not in the rendered queue, this method will check all frames. 528 void notifyOfRenderedFrames( 529 bool dropIncomplete = false, FrameRenderTracker::Info *until = NULL); 530 531 // Pass |expectedFormat| to print a warning if the format differs from it. 532 // Using sp<> instead of const sp<>& because expectedFormat is likely the current mOutputFormat 533 // which will get updated inside. 534 void onOutputFormatChanged(sp<const AMessage> expectedFormat = NULL); 535 void addKeyFormatChangesToRenderBufferNotification(sp<AMessage> ¬ify); 536 void sendFormatChange(); 537 538 status_t getPortFormat(OMX_U32 portIndex, sp<AMessage> ¬ify); 539 540 void signalError( 541 OMX_ERRORTYPE error = OMX_ErrorUndefined, 542 status_t internalError = UNKNOWN_ERROR); 543 544 static bool describeDefaultColorFormat(DescribeColorFormat2Params &describeParams); 545 static bool describeColorFormat( 546 const sp<IOMX> &omx, IOMX::node_id node, 547 DescribeColorFormat2Params &describeParams); 548 549 status_t requestIDRFrame(); 550 status_t setParameters(const sp<AMessage> ¶ms); 551 552 // Send EOS on input stream. 553 void onSignalEndOfInputStream(); 554 555 DISALLOW_EVIL_CONSTRUCTORS(ACodec); 556 }; 557 558 } // namespace android 559 560 #endif // A_CODEC_H_ 561