1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef GRAPHIC_BUFFER_SOURCE_H_ 18 19 #define GRAPHIC_BUFFER_SOURCE_H_ 20 21 #include <gui/IGraphicBufferProducer.h> 22 #include <gui/BufferQueue.h> 23 #include <utils/RefBase.h> 24 25 #include <VideoAPI.h> 26 #include <media/IOMX.h> 27 #include <media/OMXFenceParcelable.h> 28 #include <media/stagefright/foundation/ABase.h> 29 #include <media/stagefright/foundation/AHandlerReflector.h> 30 #include <media/stagefright/foundation/ALooper.h> 31 32 #include <android/BnGraphicBufferSource.h> 33 #include <android/BnOMXBufferSource.h> 34 35 #include "IOmxNodeWrapper.h" 36 37 namespace android { 38 39 using ::android::binder::Status; 40 41 struct FrameDropper; 42 43 /* 44 * This class is used to feed OMX codecs from a Surface via BufferQueue or 45 * HW producer. 46 * 47 * Instances of the class don't run on a dedicated thread. Instead, 48 * various events trigger data movement: 49 * 50 * - Availability of a new frame of data from the BufferQueue (notified 51 * via the onFrameAvailable callback). 52 * - The return of a codec buffer (via OnEmptyBufferDone). 53 * - Application signaling end-of-stream. 54 * - Transition to or from "executing" state. 55 * 56 * Frames of data (and, perhaps, the end-of-stream indication) can arrive 57 * before the codec is in the "executing" state, so we need to queue 58 * things up until we're ready to go. 59 * 60 * The GraphicBufferSource can be configure dynamically to discard frames 61 * from the source: 62 * 63 * - if their timestamp is less than a start time 64 * - if the source is suspended or stopped and the suspend/stop-time is reached 65 * - if EOS was signaled 66 * - if there is no encoder connected to it 67 * 68 * The source, furthermore, may choose to not encode (drop) frames if: 69 * 70 * - to throttle the frame rate (keep it under a certain limit) 71 * 72 * Finally the source may optionally hold onto the last non-discarded frame 73 * (even if it was dropped) to reencode it after an interval if no further 74 * frames are sent by the producer. 75 */ 76 class GraphicBufferSource : public BufferQueue::ConsumerListener { 77 public: 78 GraphicBufferSource(); 79 80 virtual ~GraphicBufferSource(); 81 82 // We can't throw an exception if the constructor fails, so we just set 83 // this and require that the caller test the value. 84 status_t initCheck() const { 85 return mInitCheck; 86 } 87 88 // Returns the handle to the producer side of the BufferQueue. Buffers 89 // queued on this will be received by GraphicBufferSource. 90 sp<IGraphicBufferProducer> getIGraphicBufferProducer() const { 91 return mProducer; 92 } 93 94 // OmxBufferSource interface 95 // ------------------------------ 96 97 // This is called when OMX transitions to OMX_StateExecuting, which means 98 // we can start handing it buffers. If we already have buffers of data 99 // sitting in the BufferQueue, this will send them to the codec. 100 Status onOmxExecuting(); 101 102 // This is called when OMX transitions to OMX_StateIdle, indicating that 103 // the codec is meant to return all buffers back to the client for them 104 // to be freed. Do NOT submit any more buffers to the component. 105 Status onOmxIdle(); 106 107 // This is called when OMX transitions to OMX_StateLoaded, indicating that 108 // we are shutting down. 109 Status onOmxLoaded(); 110 111 // A "codec buffer", i.e. a buffer that can be used to pass data into 112 // the encoder, has been allocated. (This call does not call back into 113 // OMXNodeInstance.) 114 Status onInputBufferAdded(int32_t bufferId); 115 116 // Called from OnEmptyBufferDone. If we have a BQ buffer available, 117 // fill it with a new frame of data; otherwise, just mark it as available. 118 Status onInputBufferEmptied(int32_t bufferId, int fenceFd); 119 120 // IGraphicBufferSource interface 121 // ------------------------------ 122 123 // Configure the buffer source to be used with an OMX node with the default 124 // data space. 125 status_t configure( 126 const sp<IOmxNodeWrapper> &omxNode, 127 int32_t dataSpace, 128 int32_t bufferCount, 129 uint32_t frameWidth, 130 uint32_t frameHeight, 131 uint32_t consumerUsage); 132 133 // This is called after the last input frame has been submitted or buffer 134 // timestamp is greater or equal than stopTimeUs. We need to submit an empty 135 // buffer with the EOS flag set. If we don't have a codec buffer ready, 136 // we just set the mEndOfStream flag. 137 status_t signalEndOfInputStream(); 138 139 // If suspend is true, all incoming buffers (including those currently 140 // in the BufferQueue) with timestamp larger than timeUs will be discarded 141 // until the suspension is lifted. If suspend is false, all incoming buffers 142 // including those currently in the BufferQueue) with timestamp larger than 143 // timeUs will be processed. timeUs uses SYSTEM_TIME_MONOTONIC time base. 144 status_t setSuspend(bool suspend, int64_t timeUs); 145 146 // Specifies the interval after which we requeue the buffer previously 147 // queued to the encoder. This is useful in the case of surface flinger 148 // providing the input surface if the resulting encoded stream is to 149 // be displayed "live". If we were not to push through the extra frame 150 // the decoder on the remote end would be unable to decode the latest frame. 151 // This API must be called before transitioning the encoder to "executing" 152 // state and once this behaviour is specified it cannot be reset. 153 status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs); 154 155 // Sets the input buffer timestamp offset. 156 // When set, the sample's timestamp will be adjusted with the timeOffsetUs. 157 status_t setTimeOffsetUs(int64_t timeOffsetUs); 158 159 // When set, the max frame rate fed to the encoder will be capped at maxFps. 160 status_t setMaxFps(float maxFps); 161 162 // Sets the time lapse (or slow motion) parameters. 163 // When set, the sample's timestamp will be modified to playback framerate, 164 // and capture timestamp will be modified to capture rate. 165 status_t setTimeLapseConfig(double fps, double captureFps); 166 167 // Sets the start time us (in system time), samples before which should 168 // be dropped and not submitted to encoder 169 status_t setStartTimeUs(int64_t startTimeUs); 170 171 // Sets the stop time us (in system time), samples after which should be dropped 172 // and not submitted to encoder. timeUs uses SYSTEM_TIME_MONOTONIC time base. 173 status_t setStopTimeUs(int64_t stopTimeUs); 174 175 // Sets the desired color aspects, e.g. to be used when producer does not specify a dataspace. 176 status_t setColorAspects(int32_t aspectsPacked); 177 178 protected: 179 // BQ::ConsumerListener interface 180 // ------------------------------ 181 182 // BufferQueue::ConsumerListener interface, called when a new frame of 183 // data is available. If we're executing and a codec buffer is 184 // available, we acquire the buffer, copy the GraphicBuffer reference 185 // into the codec buffer, and call Empty[This]Buffer. If we're not yet 186 // executing or there's no codec buffer available, we just increment 187 // mNumFramesAvailable and return. 188 void onFrameAvailable(const BufferItem& item) override; 189 190 // BufferQueue::ConsumerListener interface, called when the client has 191 // released one or more GraphicBuffers. We clear out the appropriate 192 // set of mBufferSlot entries. 193 void onBuffersReleased() override; 194 195 // BufferQueue::ConsumerListener interface, called when the client has 196 // changed the sideband stream. GraphicBufferSource doesn't handle sideband 197 // streams so this is a no-op (and should never be called). 198 void onSidebandStreamChanged() override; 199 200 private: 201 // Lock, covers all member variables. 202 mutable Mutex mMutex; 203 204 // Used to report constructor failure. 205 status_t mInitCheck; 206 207 // Graphic buffer reference objects 208 // -------------------------------- 209 210 // These are used to keep a shared reference to GraphicBuffers and gralloc handles owned by the 211 // GraphicBufferSource as well as to manage the cache slots. Separate references are owned by 212 // the buffer cache (controlled by the buffer queue/buffer producer) and the codec. 213 214 // When we get a buffer from the producer (BQ) it designates them to be cached into specific 215 // slots. Each slot owns a shared reference to the graphic buffer (we track these using 216 // CachedBuffer) that is in that slot, but the producer controls the slots. 217 struct CachedBuffer; 218 219 // When we acquire a buffer, we must release it back to the producer once we (or the codec) 220 // no longer uses it (as long as the buffer is still in the cache slot). We use shared 221 // AcquiredBuffer instances for this purpose - and we call release buffer when the last 222 // reference is relinquished. 223 struct AcquiredBuffer; 224 225 // We also need to keep some extra metadata (other than the buffer reference) for acquired 226 // buffers. These are tracked in VideoBuffer struct. 227 struct VideoBuffer { 228 std::shared_ptr<AcquiredBuffer> mBuffer; 229 nsecs_t mTimestampNs; 230 android_dataspace_t mDataspace; 231 }; 232 233 // Cached and aquired buffers 234 // -------------------------------- 235 236 typedef int slot_id; 237 238 // Maps a slot to the cached buffer in that slot 239 KeyedVector<slot_id, std::shared_ptr<CachedBuffer>> mBufferSlots; 240 241 // Queue of buffers acquired in chronological order that are not yet submitted to the codec 242 List<VideoBuffer> mAvailableBuffers; 243 244 // Number of buffers that have been signaled by the producer that they are available, but 245 // we've been unable to acquire them due to our max acquire count 246 int32_t mNumAvailableUnacquiredBuffers; 247 248 // Number of frames acquired from consumer (debug only) 249 // (as in aquireBuffer called, and release needs to be called) 250 int32_t mNumOutstandingAcquires; 251 252 // Acquire a buffer from the BQ and store it in |item| if successful 253 // \return OK on success, or error on failure. 254 status_t acquireBuffer_l(VideoBuffer *item); 255 256 // Called when a buffer was acquired from the producer 257 void onBufferAcquired_l(const VideoBuffer &buffer); 258 259 // marks the buffer at the slot no longer cached, and accounts for the outstanding 260 // acquire count 261 void discardBufferInSlot_l(slot_id i); 262 263 // marks the buffer at the slot index no longer cached, and accounts for the outstanding 264 // acquire count 265 void discardBufferAtSlotIndex_l(ssize_t bsi); 266 267 // release all acquired and unacquired available buffers 268 // This method will return if it fails to acquire an unacquired available buffer, which will 269 // leave mNumAvailableUnacquiredBuffers positive on return. 270 void releaseAllAvailableBuffers_l(); 271 272 // returns whether we have any available buffers (acquired or not-yet-acquired) 273 bool haveAvailableBuffers_l() const { 274 return !mAvailableBuffers.empty() || mNumAvailableUnacquiredBuffers > 0; 275 } 276 277 // Codec buffers 278 // ------------- 279 280 // When we queue buffers to the encoder, we must hold the references to the graphic buffers 281 // in those buffers - as the producer may free the slots. 282 283 typedef int32_t codec_buffer_id; 284 285 // set of codec buffer ID-s of buffers available to fill 286 List<codec_buffer_id> mFreeCodecBuffers; 287 288 // maps codec buffer ID-s to buffer info submitted to the codec. Used to keep a reference for 289 // the graphics buffer. 290 KeyedVector<codec_buffer_id, std::shared_ptr<AcquiredBuffer>> mSubmittedCodecBuffers; 291 292 // Processes the next acquired frame. If there is no available codec buffer, it returns false 293 // without any further action. 294 // 295 // Otherwise, it consumes the next acquired frame and determines if it needs to be discarded or 296 // dropped. If neither are needed, it submits it to the codec. It also saves the latest 297 // non-dropped frame and submits it for repeat encoding (if this is enabled). 298 // 299 // \require there must be an acquired frame (i.e. we're in the onFrameAvailable callback, 300 // or if we're in codecBufferEmptied and mNumFramesAvailable is nonzero). 301 // \require codec must be executing 302 // \returns true if acquired (and handled) the next frame. Otherwise, false. 303 bool fillCodecBuffer_l(); 304 305 // Calculates the media timestamp for |item| and on success it submits the buffer to the codec, 306 // while also keeping a reference for it in mSubmittedCodecBuffers. 307 // Returns UNKNOWN_ERROR if the buffer was not submitted due to buffer timestamp. Otherwise, 308 // it returns any submit success or error value returned by the codec. 309 status_t submitBuffer_l(const VideoBuffer &item); 310 311 // Submits an empty buffer, with the EOS flag set if there is an available codec buffer and 312 // sets mEndOfStreamSent flag. Does nothing if there is no codec buffer available. 313 void submitEndOfInputStream_l(); 314 315 // Set to true if we want to send end-of-stream after we run out of available frames from the 316 // producer 317 bool mEndOfStream; 318 319 // Flag that the EOS was submitted to the encoder 320 bool mEndOfStreamSent; 321 322 // Dataspace for the last frame submitted to the codec 323 android_dataspace mLastDataspace; 324 325 // Default color aspects for this source 326 int32_t mDefaultColorAspectsPacked; 327 328 // called when the data space of the input buffer changes 329 void onDataspaceChanged_l(android_dataspace dataspace, android_pixel_format pixelFormat); 330 331 // Pointer back to the Omx node that created us. We send buffers here. 332 sp<IOmxNodeWrapper> mOMXNode; 333 334 // Set by omxExecuting() / omxIdling(). 335 bool mExecuting; 336 337 bool mSuspended; 338 339 // returns true if this source is unconditionally discarding acquired buffers at the moment 340 // regardless of the metadata of those buffers 341 bool areWeDiscardingAvailableBuffers_l(); 342 343 // Our BufferQueue interfaces. mProducer is passed to the producer through 344 // getIGraphicBufferProducer, and mConsumer is used internally to retrieve 345 // the buffers queued by the producer. 346 sp<IGraphicBufferProducer> mProducer; 347 sp<IGraphicBufferConsumer> mConsumer; 348 349 // The time to stop sending buffers. 350 int64_t mStopTimeUs; 351 352 struct ActionItem { 353 typedef enum { 354 PAUSE, 355 RESUME, 356 STOP 357 } ActionType; 358 ActionType mAction; 359 int64_t mActionTimeUs; 360 }; 361 362 // Maintain last action timestamp to ensure all the action timestamps are 363 // monotonically increasing. 364 int64_t mLastActionTimeUs; 365 366 // An action queue that queue up all the actions sent to GraphicBufferSource. 367 // STOP action should only show up at the end of the list as all the actions 368 // after a STOP action will be discarded. mActionQueue is protected by mMutex. 369 List<ActionItem> mActionQueue; 370 371 //// 372 friend struct AHandlerReflector<GraphicBufferSource>; 373 374 enum { 375 kWhatRepeatLastFrame, ///< queue last frame for reencoding 376 }; 377 enum { 378 kRepeatLastFrameCount = 10, 379 }; 380 381 int64_t mSkipFramesBeforeNs; 382 383 sp<FrameDropper> mFrameDropper; 384 385 sp<ALooper> mLooper; 386 sp<AHandlerReflector<GraphicBufferSource> > mReflector; 387 388 // Repeat last frame feature 389 // ------------------------- 390 // configuration parameter: repeat interval for frame repeating (<0 if repeating is disabled) 391 int64_t mFrameRepeatIntervalUs; 392 393 // current frame repeat generation - used to cancel a pending frame repeat 394 int32_t mRepeatLastFrameGeneration; 395 396 // number of times to repeat latest frame (0 = none) 397 int32_t mOutstandingFrameRepeatCount; 398 399 // The previous buffer should've been repeated but 400 // no codec buffer was available at the time. 401 bool mFrameRepeatBlockedOnCodecBuffer; 402 403 // hold a reference to the last acquired (and not discarded) frame for frame repeating 404 VideoBuffer mLatestBuffer; 405 406 // queue last frame for reencode after the repeat interval. 407 void queueFrameRepeat_l(); 408 409 // save |item| as the latest buffer and queue it for reencode (repeat) 410 void setLatestBuffer_l(const VideoBuffer &item); 411 412 // submit last frame to encoder and queue it for reencode 413 // \return true if buffer was submitted, false if it wasn't (e.g. source is suspended, there 414 // is no available codec buffer) 415 bool repeatLatestBuffer_l(); 416 417 // Time lapse / slow motion configuration 418 // -------------------------------------- 419 420 // desired frame rate for encoding - value <= 0 if undefined 421 double mFps; 422 423 // desired frame rate for capture - value <= 0 if undefined 424 double mCaptureFps; 425 426 // Time lapse mode is enabled if the capture frame rate is defined and it is 427 // smaller than half the encoding frame rate (if defined). In this mode, 428 // frames that come in between the capture interval (the reciprocal of the 429 // capture frame rate) are dropped and the encoding timestamp is adjusted to 430 // match the desired encoding frame rate. 431 // 432 // Slow motion mode is enabled if both encoding and capture frame rates are 433 // defined and the encoding frame rate is less than half the capture frame 434 // rate. In this mode, the source is expected to produce frames with an even 435 // timestamp interval (after rounding) with the configured capture fps. The 436 // first source timestamp is used as the source base time. Afterwards, the 437 // timestamp of each source frame is snapped to the nearest expected capture 438 // timestamp and scaled to match the configured encoding frame rate. 439 440 // These modes must be enabled before using this source. 441 442 // adjusted capture timestamp of the base frame 443 int64_t mBaseCaptureUs; 444 445 // adjusted encoding timestamp of the base frame 446 int64_t mBaseFrameUs; 447 448 // number of frames from the base time 449 int64_t mFrameCount; 450 451 // adjusted capture timestamp for previous frame (negative if there were 452 // none) 453 int64_t mPrevCaptureUs; 454 455 // adjusted media timestamp for previous frame (negative if there were none) 456 int64_t mPrevFrameUs; 457 458 // desired offset between media time and capture time 459 int64_t mInputBufferTimeOffsetUs; 460 461 // Calculates and outputs the timestamp to use for a buffer with a specific buffer timestamp 462 // |bufferTimestampNs|. Returns false on failure (buffer too close or timestamp is moving 463 // backwards). Otherwise, stores the media timestamp in |*codecTimeUs| and returns true. 464 // 465 // This method takes into account the start time offset and any time lapse or slow motion time 466 // adjustment requests. 467 bool calculateCodecTimestamp_l(nsecs_t bufferTimeNs, int64_t *codecTimeUs); 468 469 void onMessageReceived(const sp<AMessage> &msg); 470 471 DISALLOW_EVIL_CONSTRUCTORS(GraphicBufferSource); 472 }; 473 474 } // namespace android 475 476 #endif // GRAPHIC_BUFFER_SOURCE_H_ 477