1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef GRAPHIC_BUFFER_SOURCE_H_ 18 19 #define GRAPHIC_BUFFER_SOURCE_H_ 20 21 #include <gui/IGraphicBufferProducer.h> 22 #include <gui/BufferQueue.h> 23 #include <utils/RefBase.h> 24 25 #include <media/hardware/VideoAPI.h> 26 #include <media/IOMX.h> 27 #include <media/OMXFenceParcelable.h> 28 #include <media/stagefright/foundation/ABase.h> 29 #include <media/stagefright/foundation/AHandlerReflector.h> 30 #include <media/stagefright/foundation/ALooper.h> 31 32 #include <android/BnGraphicBufferSource.h> 33 #include <android/BnOMXBufferSource.h> 34 35 #include "IOmxNodeWrapper.h" 36 37 namespace android { 38 39 using ::android::binder::Status; 40 41 struct FrameDropper; 42 43 /* 44 * This class is used to feed OMX codecs from a Surface via BufferQueue or 45 * HW producer. 46 * 47 * Instances of the class don't run on a dedicated thread. Instead, 48 * various events trigger data movement: 49 * 50 * - Availability of a new frame of data from the BufferQueue (notified 51 * via the onFrameAvailable callback). 52 * - The return of a codec buffer (via OnEmptyBufferDone). 53 * - Application signaling end-of-stream. 54 * - Transition to or from "executing" state. 55 * 56 * Frames of data (and, perhaps, the end-of-stream indication) can arrive 57 * before the codec is in the "executing" state, so we need to queue 58 * things up until we're ready to go. 59 * 60 * The GraphicBufferSource can be configure dynamically to discard frames 61 * from the source: 62 * 63 * - if their timestamp is less than a start time 64 * - if the source is suspended or stopped and the suspend/stop-time is reached 65 * - if EOS was signaled 66 * - if there is no encoder connected to it 67 * 68 * The source, furthermore, may choose to not encode (drop) frames if: 69 * 70 * - to throttle the frame rate (keep it under a certain limit) 71 * 72 * Finally the source may optionally hold onto the last non-discarded frame 73 * (even if it was dropped) to reencode it after an interval if no further 74 * frames are sent by the producer. 75 */ 76 class GraphicBufferSource : public BufferQueue::ConsumerListener { 77 public: 78 GraphicBufferSource(); 79 80 virtual ~GraphicBufferSource(); 81 82 // We can't throw an exception if the constructor fails, so we just set 83 // this and require that the caller test the value. 84 status_t initCheck() const { 85 return mInitCheck; 86 } 87 88 // Returns the handle to the producer side of the BufferQueue. Buffers 89 // queued on this will be received by GraphicBufferSource. 90 sp<IGraphicBufferProducer> getIGraphicBufferProducer() const { 91 return mProducer; 92 } 93 94 // OmxBufferSource interface 95 // ------------------------------ 96 97 // This is called when OMX transitions to OMX_StateExecuting, which means 98 // we can start handing it buffers. If we already have buffers of data 99 // sitting in the BufferQueue, this will send them to the codec. 100 Status onOmxExecuting(); 101 102 // This is called when OMX transitions to OMX_StateIdle, indicating that 103 // the codec is meant to return all buffers back to the client for them 104 // to be freed. Do NOT submit any more buffers to the component. 105 Status onOmxIdle(); 106 107 // This is called when OMX transitions to OMX_StateLoaded, indicating that 108 // we are shutting down. 109 Status onOmxLoaded(); 110 111 // A "codec buffer", i.e. a buffer that can be used to pass data into 112 // the encoder, has been allocated. (This call does not call back into 113 // OMXNodeInstance.) 114 Status onInputBufferAdded(int32_t bufferId); 115 116 // Called from OnEmptyBufferDone. If we have a BQ buffer available, 117 // fill it with a new frame of data; otherwise, just mark it as available. 118 Status onInputBufferEmptied(int32_t bufferId, int fenceFd); 119 120 // IGraphicBufferSource interface 121 // ------------------------------ 122 123 // Configure the buffer source to be used with an OMX node with the default 124 // data space. 125 status_t configure( 126 const sp<IOmxNodeWrapper> &omxNode, 127 int32_t dataSpace, 128 int32_t bufferCount, 129 uint32_t frameWidth, 130 uint32_t frameHeight, 131 uint32_t consumerUsage); 132 133 // This is called after the last input frame has been submitted or buffer 134 // timestamp is greater or equal than stopTimeUs. We need to submit an empty 135 // buffer with the EOS flag set. If we don't have a codec buffer ready, 136 // we just set the mEndOfStream flag. 137 status_t signalEndOfInputStream(); 138 139 // If suspend is true, all incoming buffers (including those currently 140 // in the BufferQueue) with timestamp larger than timeUs will be discarded 141 // until the suspension is lifted. If suspend is false, all incoming buffers 142 // including those currently in the BufferQueue) with timestamp larger than 143 // timeUs will be processed. timeUs uses SYSTEM_TIME_MONOTONIC time base. 144 status_t setSuspend(bool suspend, int64_t timeUs); 145 146 // Specifies the interval after which we requeue the buffer previously 147 // queued to the encoder. This is useful in the case of surface flinger 148 // providing the input surface if the resulting encoded stream is to 149 // be displayed "live". If we were not to push through the extra frame 150 // the decoder on the remote end would be unable to decode the latest frame. 151 // This API must be called before transitioning the encoder to "executing" 152 // state and once this behaviour is specified it cannot be reset. 153 status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs); 154 155 // Sets the input buffer timestamp offset. 156 // When set, the sample's timestamp will be adjusted with the timeOffsetUs. 157 status_t setTimeOffsetUs(int64_t timeOffsetUs); 158 159 // When set, the max frame rate fed to the encoder will be capped at maxFps. 160 status_t setMaxFps(float maxFps); 161 162 // Sets the time lapse (or slow motion) parameters. 163 // When set, the sample's timestamp will be modified to playback framerate, 164 // and capture timestamp will be modified to capture rate. 165 status_t setTimeLapseConfig(double fps, double captureFps); 166 167 // Sets the start time us (in system time), samples before which should 168 // be dropped and not submitted to encoder 169 status_t setStartTimeUs(int64_t startTimeUs); 170 171 // Sets the stop time us (in system time), samples after which should be dropped 172 // and not submitted to encoder. timeUs uses SYSTEM_TIME_MONOTONIC time base. 173 status_t setStopTimeUs(int64_t stopTimeUs); 174 175 // Gets the stop time offset in us. This is the time offset between latest buffer 176 // time and the stopTimeUs. If stop time is not set, INVALID_OPERATION will be returned. 177 // If return is OK, *stopTimeOffsetUs will contain the valid offset. Otherwise, 178 // *stopTimeOffsetUs will not be modified. Positive stopTimeOffsetUs means buffer time 179 // larger than stopTimeUs. 180 status_t getStopTimeOffsetUs(int64_t *stopTimeOffsetUs); 181 182 // Sets the desired color aspects, e.g. to be used when producer does not specify a dataspace. 183 status_t setColorAspects(int32_t aspectsPacked); 184 185 protected: 186 // BQ::ConsumerListener interface 187 // ------------------------------ 188 189 // BufferQueue::ConsumerListener interface, called when a new frame of 190 // data is available. If we're executing and a codec buffer is 191 // available, we acquire the buffer, copy the GraphicBuffer reference 192 // into the codec buffer, and call Empty[This]Buffer. If we're not yet 193 // executing or there's no codec buffer available, we just increment 194 // mNumFramesAvailable and return. 195 void onFrameAvailable(const BufferItem& item) override; 196 197 // BufferQueue::ConsumerListener interface, called when the client has 198 // released one or more GraphicBuffers. We clear out the appropriate 199 // set of mBufferSlot entries. 200 void onBuffersReleased() override; 201 202 // BufferQueue::ConsumerListener interface, called when the client has 203 // changed the sideband stream. GraphicBufferSource doesn't handle sideband 204 // streams so this is a no-op (and should never be called). 205 void onSidebandStreamChanged() override; 206 207 private: 208 // Lock, covers all member variables. 209 mutable Mutex mMutex; 210 211 // Used to report constructor failure. 212 status_t mInitCheck; 213 214 // Graphic buffer reference objects 215 // -------------------------------- 216 217 // These are used to keep a shared reference to GraphicBuffers and gralloc handles owned by the 218 // GraphicBufferSource as well as to manage the cache slots. Separate references are owned by 219 // the buffer cache (controlled by the buffer queue/buffer producer) and the codec. 220 221 // When we get a buffer from the producer (BQ) it designates them to be cached into specific 222 // slots. Each slot owns a shared reference to the graphic buffer (we track these using 223 // CachedBuffer) that is in that slot, but the producer controls the slots. 224 struct CachedBuffer; 225 226 // When we acquire a buffer, we must release it back to the producer once we (or the codec) 227 // no longer uses it (as long as the buffer is still in the cache slot). We use shared 228 // AcquiredBuffer instances for this purpose - and we call release buffer when the last 229 // reference is relinquished. 230 struct AcquiredBuffer; 231 232 // We also need to keep some extra metadata (other than the buffer reference) for acquired 233 // buffers. These are tracked in VideoBuffer struct. 234 struct VideoBuffer { 235 std::shared_ptr<AcquiredBuffer> mBuffer; 236 nsecs_t mTimestampNs; 237 android_dataspace_t mDataspace; 238 }; 239 240 // Cached and aquired buffers 241 // -------------------------------- 242 243 typedef int slot_id; 244 245 // Maps a slot to the cached buffer in that slot 246 KeyedVector<slot_id, std::shared_ptr<CachedBuffer>> mBufferSlots; 247 248 // Queue of buffers acquired in chronological order that are not yet submitted to the codec 249 List<VideoBuffer> mAvailableBuffers; 250 251 // Number of buffers that have been signaled by the producer that they are available, but 252 // we've been unable to acquire them due to our max acquire count 253 int32_t mNumAvailableUnacquiredBuffers; 254 255 // Number of frames acquired from consumer (debug only) 256 // (as in aquireBuffer called, and release needs to be called) 257 int32_t mNumOutstandingAcquires; 258 259 // Acquire a buffer from the BQ and store it in |item| if successful 260 // \return OK on success, or error on failure. 261 status_t acquireBuffer_l(VideoBuffer *item); 262 263 // Called when a buffer was acquired from the producer 264 void onBufferAcquired_l(const VideoBuffer &buffer); 265 266 // marks the buffer at the slot no longer cached, and accounts for the outstanding 267 // acquire count. Returns true if the slot was populated; otherwise, false. 268 bool discardBufferInSlot_l(slot_id i); 269 270 // marks the buffer at the slot index no longer cached, and accounts for the outstanding 271 // acquire count 272 void discardBufferAtSlotIndex_l(ssize_t bsi); 273 274 // release all acquired and unacquired available buffers 275 // This method will return if it fails to acquire an unacquired available buffer, which will 276 // leave mNumAvailableUnacquiredBuffers positive on return. 277 void releaseAllAvailableBuffers_l(); 278 279 // returns whether we have any available buffers (acquired or not-yet-acquired) 280 bool haveAvailableBuffers_l() const { 281 return !mAvailableBuffers.empty() || mNumAvailableUnacquiredBuffers > 0; 282 } 283 284 // Codec buffers 285 // ------------- 286 287 // When we queue buffers to the encoder, we must hold the references to the graphic buffers 288 // in those buffers - as the producer may free the slots. 289 290 typedef int32_t codec_buffer_id; 291 292 // set of codec buffer ID-s of buffers available to fill 293 List<codec_buffer_id> mFreeCodecBuffers; 294 295 // maps codec buffer ID-s to buffer info submitted to the codec. Used to keep a reference for 296 // the graphics buffer. 297 KeyedVector<codec_buffer_id, std::shared_ptr<AcquiredBuffer>> mSubmittedCodecBuffers; 298 299 // Processes the next acquired frame. If there is no available codec buffer, it returns false 300 // without any further action. 301 // 302 // Otherwise, it consumes the next acquired frame and determines if it needs to be discarded or 303 // dropped. If neither are needed, it submits it to the codec. It also saves the latest 304 // non-dropped frame and submits it for repeat encoding (if this is enabled). 305 // 306 // \require there must be an acquired frame (i.e. we're in the onFrameAvailable callback, 307 // or if we're in codecBufferEmptied and mNumFramesAvailable is nonzero). 308 // \require codec must be executing 309 // \returns true if acquired (and handled) the next frame. Otherwise, false. 310 bool fillCodecBuffer_l(); 311 312 // Calculates the media timestamp for |item| and on success it submits the buffer to the codec, 313 // while also keeping a reference for it in mSubmittedCodecBuffers. 314 // Returns UNKNOWN_ERROR if the buffer was not submitted due to buffer timestamp. Otherwise, 315 // it returns any submit success or error value returned by the codec. 316 status_t submitBuffer_l(const VideoBuffer &item); 317 318 // Submits an empty buffer, with the EOS flag set if there is an available codec buffer and 319 // sets mEndOfStreamSent flag. Does nothing if there is no codec buffer available. 320 void submitEndOfInputStream_l(); 321 322 // Set to true if we want to send end-of-stream after we run out of available frames from the 323 // producer 324 bool mEndOfStream; 325 326 // Flag that the EOS was submitted to the encoder 327 bool mEndOfStreamSent; 328 329 // Dataspace for the last frame submitted to the codec 330 android_dataspace mLastDataspace; 331 332 // Default color aspects for this source 333 int32_t mDefaultColorAspectsPacked; 334 335 // called when the data space of the input buffer changes 336 void onDataspaceChanged_l(android_dataspace dataspace, android_pixel_format pixelFormat); 337 338 // Pointer back to the Omx node that created us. We send buffers here. 339 sp<IOmxNodeWrapper> mOMXNode; 340 341 // Set by omxExecuting() / omxIdling(). 342 bool mExecuting; 343 344 bool mSuspended; 345 346 // returns true if this source is unconditionally discarding acquired buffers at the moment 347 // regardless of the metadata of those buffers 348 bool areWeDiscardingAvailableBuffers_l(); 349 350 int64_t mLastFrameTimestampUs; 351 352 // Our BufferQueue interfaces. mProducer is passed to the producer through 353 // getIGraphicBufferProducer, and mConsumer is used internally to retrieve 354 // the buffers queued by the producer. 355 sp<IGraphicBufferProducer> mProducer; 356 sp<IGraphicBufferConsumer> mConsumer; 357 358 // The time to stop sending buffers. 359 int64_t mStopTimeUs; 360 361 struct ActionItem { 362 typedef enum { 363 PAUSE, 364 RESUME, 365 STOP 366 } ActionType; 367 ActionType mAction; 368 int64_t mActionTimeUs; 369 }; 370 371 // Maintain last action timestamp to ensure all the action timestamps are 372 // monotonically increasing. 373 int64_t mLastActionTimeUs; 374 375 // An action queue that queue up all the actions sent to GraphicBufferSource. 376 // STOP action should only show up at the end of the list as all the actions 377 // after a STOP action will be discarded. mActionQueue is protected by mMutex. 378 List<ActionItem> mActionQueue; 379 380 //// 381 friend struct AHandlerReflector<GraphicBufferSource>; 382 383 enum { 384 kWhatRepeatLastFrame, ///< queue last frame for reencoding 385 }; 386 enum { 387 kRepeatLastFrameCount = 10, 388 }; 389 390 int64_t mSkipFramesBeforeNs; 391 392 sp<FrameDropper> mFrameDropper; 393 394 sp<ALooper> mLooper; 395 sp<AHandlerReflector<GraphicBufferSource> > mReflector; 396 397 // Repeat last frame feature 398 // ------------------------- 399 // configuration parameter: repeat interval for frame repeating (<0 if repeating is disabled) 400 int64_t mFrameRepeatIntervalUs; 401 402 // current frame repeat generation - used to cancel a pending frame repeat 403 int32_t mRepeatLastFrameGeneration; 404 405 // number of times to repeat latest frame (0 = none) 406 int32_t mOutstandingFrameRepeatCount; 407 408 // The previous buffer should've been repeated but 409 // no codec buffer was available at the time. 410 bool mFrameRepeatBlockedOnCodecBuffer; 411 412 // hold a reference to the last acquired (and not discarded) frame for frame repeating 413 VideoBuffer mLatestBuffer; 414 415 // queue last frame for reencode after the repeat interval. 416 void queueFrameRepeat_l(); 417 418 // save |item| as the latest buffer and queue it for reencode (repeat) 419 void setLatestBuffer_l(const VideoBuffer &item); 420 421 // submit last frame to encoder and queue it for reencode 422 // \return true if buffer was submitted, false if it wasn't (e.g. source is suspended, there 423 // is no available codec buffer) 424 bool repeatLatestBuffer_l(); 425 426 // Time lapse / slow motion configuration 427 // -------------------------------------- 428 429 // desired frame rate for encoding - value <= 0 if undefined 430 double mFps; 431 432 // desired frame rate for capture - value <= 0 if undefined 433 double mCaptureFps; 434 435 // Time lapse mode is enabled if the capture frame rate is defined and it is 436 // smaller than half the encoding frame rate (if defined). In this mode, 437 // frames that come in between the capture interval (the reciprocal of the 438 // capture frame rate) are dropped and the encoding timestamp is adjusted to 439 // match the desired encoding frame rate. 440 // 441 // Slow motion mode is enabled if both encoding and capture frame rates are 442 // defined and the encoding frame rate is less than half the capture frame 443 // rate. In this mode, the source is expected to produce frames with an even 444 // timestamp interval (after rounding) with the configured capture fps. The 445 // first source timestamp is used as the source base time. Afterwards, the 446 // timestamp of each source frame is snapped to the nearest expected capture 447 // timestamp and scaled to match the configured encoding frame rate. 448 449 // These modes must be enabled before using this source. 450 451 // adjusted capture timestamp of the base frame 452 int64_t mBaseCaptureUs; 453 454 // adjusted encoding timestamp of the base frame 455 int64_t mBaseFrameUs; 456 457 // number of frames from the base time 458 int64_t mFrameCount; 459 460 // adjusted capture timestamp for previous frame (negative if there were 461 // none) 462 int64_t mPrevCaptureUs; 463 464 // adjusted media timestamp for previous frame (negative if there were none) 465 int64_t mPrevFrameUs; 466 467 // desired offset between media time and capture time 468 int64_t mInputBufferTimeOffsetUs; 469 470 // Calculates and outputs the timestamp to use for a buffer with a specific buffer timestamp 471 // |bufferTimestampNs|. Returns false on failure (buffer too close or timestamp is moving 472 // backwards). Otherwise, stores the media timestamp in |*codecTimeUs| and returns true. 473 // 474 // This method takes into account the start time offset and any time lapse or slow motion time 475 // adjustment requests. 476 bool calculateCodecTimestamp_l(nsecs_t bufferTimeNs, int64_t *codecTimeUs); 477 478 void onMessageReceived(const sp<AMessage> &msg); 479 480 DISALLOW_EVIL_CONSTRUCTORS(GraphicBufferSource); 481 }; 482 483 } // namespace android 484 485 #endif // GRAPHIC_BUFFER_SOURCE_H_ 486