1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef GRAPHIC_BUFFER_SOURCE_H_ 18 19 #define GRAPHIC_BUFFER_SOURCE_H_ 20 21 #include <binder/Status.h> 22 #include <gui/BufferQueue.h> 23 #include <gui/IGraphicBufferProducer.h> 24 #include <utils/RefBase.h> 25 26 #include <media/hardware/VideoAPI.h> 27 #include <media/stagefright/foundation/ABase.h> 28 #include <media/stagefright/foundation/AHandlerReflector.h> 29 #include <media/stagefright/foundation/ALooper.h> 30 #include <media/stagefright/bqhelper/ComponentWrapper.h> 31 32 namespace android { 33 34 using ::android::binder::Status; 35 36 struct FrameDropper; 37 38 /* 39 * This class is used to feed codecs from a Surface via BufferQueue or 40 * HW producer. 41 * 42 * Instances of the class don't run on a dedicated thread. Instead, 43 * various events trigger data movement: 44 * 45 * - Availability of a new frame of data from the BufferQueue (notified 46 * via the onFrameAvailable callback). 47 * - The return of a codec buffer. 48 * - Application signaling end-of-stream. 49 * - Transition to or from "executing" state. 50 * 51 * Frames of data (and, perhaps, the end-of-stream indication) can arrive 52 * before the codec is in the "executing" state, so we need to queue 53 * things up until we're ready to go. 54 * 55 * The GraphicBufferSource can be configure dynamically to discard frames 56 * from the source: 57 * 58 * - if their timestamp is less than a start time 59 * - if the source is suspended or stopped and the suspend/stop-time is reached 60 * - if EOS was signaled 61 * - if there is no encoder connected to it 62 * 63 * The source, furthermore, may choose to not encode (drop) frames if: 64 * 65 * - to throttle the frame rate (keep it under a certain limit) 66 * 67 * Finally the source may optionally hold onto the last non-discarded frame 68 * (even if it was dropped) to reencode it after an interval if no further 69 * frames are sent by the producer. 70 */ 71 class GraphicBufferSource : public BufferQueue::ConsumerListener { 72 public: 73 GraphicBufferSource(); 74 75 virtual ~GraphicBufferSource(); 76 77 // We can't throw an exception if the constructor fails, so we just set 78 // this and require that the caller test the value. 79 status_t initCheck() const { 80 return mInitCheck; 81 } 82 83 // Returns the handle to the producer side of the BufferQueue. Buffers 84 // queued on this will be received by GraphicBufferSource. 85 sp<IGraphicBufferProducer> getIGraphicBufferProducer() const { 86 return mProducer; 87 } 88 89 // This is called when component transitions to running state, which means 90 // we can start handing it buffers. If we already have buffers of data 91 // sitting in the BufferQueue, this will send them to the codec. 92 Status start(); 93 94 // This is called when component transitions to stopped, indicating that 95 // the codec is meant to return all buffers back to the client for them 96 // to be freed. Do NOT submit any more buffers to the component. 97 Status stop(); 98 99 // This is called when component transitions to released, indicating that 100 // we are shutting down. 101 Status release(); 102 103 // A "codec buffer", i.e. a buffer that can be used to pass data into 104 // the encoder, has been allocated. (This call does not call back into 105 // component.) 106 Status onInputBufferAdded(int32_t bufferId); 107 108 // Called when encoder is no longer using the buffer. If we have a BQ 109 // buffer available, fill it with a new frame of data; otherwise, just mark 110 // it as available. 111 Status onInputBufferEmptied(int32_t bufferId, int fenceFd); 112 113 // IGraphicBufferSource interface 114 // ------------------------------ 115 116 // Configure the buffer source to be used with a component with the default 117 // data space. 118 status_t configure( 119 const sp<ComponentWrapper> &component, 120 int32_t dataSpace, 121 int32_t bufferCount, 122 uint32_t frameWidth, 123 uint32_t frameHeight, 124 uint32_t consumerUsage); 125 126 // This is called after the last input frame has been submitted or buffer 127 // timestamp is greater or equal than stopTimeUs. We need to submit an empty 128 // buffer with the EOS flag set. If we don't have a codec buffer ready, 129 // we just set the mEndOfStream flag. 130 status_t signalEndOfInputStream(); 131 132 // If suspend is true, all incoming buffers (including those currently 133 // in the BufferQueue) with timestamp larger than timeUs will be discarded 134 // until the suspension is lifted. If suspend is false, all incoming buffers 135 // including those currently in the BufferQueue) with timestamp larger than 136 // timeUs will be processed. timeUs uses SYSTEM_TIME_MONOTONIC time base. 137 status_t setSuspend(bool suspend, int64_t timeUs); 138 139 // Specifies the interval after which we requeue the buffer previously 140 // queued to the encoder. This is useful in the case of surface flinger 141 // providing the input surface if the resulting encoded stream is to 142 // be displayed "live". If we were not to push through the extra frame 143 // the decoder on the remote end would be unable to decode the latest frame. 144 // This API must be called before transitioning the encoder to "executing" 145 // state and once this behaviour is specified it cannot be reset. 146 status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs); 147 148 // Sets the input buffer timestamp offset. 149 // When set, the sample's timestamp will be adjusted with the timeOffsetUs. 150 status_t setTimeOffsetUs(int64_t timeOffsetUs); 151 152 /* 153 * Set the maximum frame rate on the source. 154 * 155 * When maxFps is a positive number, it indicates the maximum rate at which 156 * the buffers from this source will be sent to the encoder. Excessive 157 * frames will be dropped to meet the frame rate requirement. 158 * 159 * When maxFps is a negative number, any frame drop logic will be disabled 160 * and all frames from this source will be sent to the encoder, even when 161 * the timestamp goes backwards. Note that some components may still drop 162 * out-of-order frames silently, so this usually has to be used in 163 * conjunction with OMXNodeInstance::setMaxPtsGapUs() workaround. 164 * 165 * When maxFps is 0, this call will fail with BAD_VALUE. 166 */ 167 status_t setMaxFps(float maxFps); 168 169 // Sets the time lapse (or slow motion) parameters. 170 // When set, the sample's timestamp will be modified to playback framerate, 171 // and capture timestamp will be modified to capture rate. 172 status_t setTimeLapseConfig(double fps, double captureFps); 173 174 // Sets the start time us (in system time), samples before which should 175 // be dropped and not submitted to encoder 176 status_t setStartTimeUs(int64_t startTimeUs); 177 178 // Sets the stop time us (in system time), samples after which should be dropped 179 // and not submitted to encoder. timeUs uses SYSTEM_TIME_MONOTONIC time base. 180 status_t setStopTimeUs(int64_t stopTimeUs); 181 182 // Gets the stop time offset in us. This is the time offset between latest buffer 183 // time and the stopTimeUs. If stop time is not set, INVALID_OPERATION will be returned. 184 // If return is OK, *stopTimeOffsetUs will contain the valid offset. Otherwise, 185 // *stopTimeOffsetUs will not be modified. Positive stopTimeOffsetUs means buffer time 186 // larger than stopTimeUs. 187 status_t getStopTimeOffsetUs(int64_t *stopTimeOffsetUs); 188 189 // Sets the desired color aspects, e.g. to be used when producer does not specify a dataspace. 190 status_t setColorAspects(int32_t aspectsPacked); 191 192 protected: 193 // BQ::ConsumerListener interface 194 // ------------------------------ 195 196 // BufferQueue::ConsumerListener interface, called when a new frame of 197 // data is available. If we're executing and a codec buffer is 198 // available, we acquire the buffer, copy the GraphicBuffer reference 199 // into the codec buffer, and call Empty[This]Buffer. If we're not yet 200 // executing or there's no codec buffer available, we just increment 201 // mNumFramesAvailable and return. 202 void onFrameAvailable(const BufferItem& item) override; 203 204 // BufferQueue::ConsumerListener interface, called when the client has 205 // released one or more GraphicBuffers. We clear out the appropriate 206 // set of mBufferSlot entries. 207 void onBuffersReleased() override; 208 209 // BufferQueue::ConsumerListener interface, called when the client has 210 // changed the sideband stream. GraphicBufferSource doesn't handle sideband 211 // streams so this is a no-op (and should never be called). 212 void onSidebandStreamChanged() override; 213 214 private: 215 // Lock, covers all member variables. 216 mutable Mutex mMutex; 217 218 // Used to report constructor failure. 219 status_t mInitCheck; 220 221 // Graphic buffer reference objects 222 // -------------------------------- 223 224 // These are used to keep a shared reference to GraphicBuffers and gralloc handles owned by the 225 // GraphicBufferSource as well as to manage the cache slots. Separate references are owned by 226 // the buffer cache (controlled by the buffer queue/buffer producer) and the codec. 227 228 // When we get a buffer from the producer (BQ) it designates them to be cached into specific 229 // slots. Each slot owns a shared reference to the graphic buffer (we track these using 230 // CachedBuffer) that is in that slot, but the producer controls the slots. 231 struct CachedBuffer; 232 233 // When we acquire a buffer, we must release it back to the producer once we (or the codec) 234 // no longer uses it (as long as the buffer is still in the cache slot). We use shared 235 // AcquiredBuffer instances for this purpose - and we call release buffer when the last 236 // reference is relinquished. 237 struct AcquiredBuffer; 238 239 // We also need to keep some extra metadata (other than the buffer reference) for acquired 240 // buffers. These are tracked in VideoBuffer struct. 241 struct VideoBuffer { 242 std::shared_ptr<AcquiredBuffer> mBuffer; 243 nsecs_t mTimestampNs; 244 android_dataspace_t mDataspace; 245 }; 246 247 // Cached and aquired buffers 248 // -------------------------------- 249 250 typedef int slot_id; 251 252 // Maps a slot to the cached buffer in that slot 253 KeyedVector<slot_id, std::shared_ptr<CachedBuffer>> mBufferSlots; 254 255 // Queue of buffers acquired in chronological order that are not yet submitted to the codec 256 List<VideoBuffer> mAvailableBuffers; 257 258 // Number of buffers that have been signaled by the producer that they are available, but 259 // we've been unable to acquire them due to our max acquire count 260 int32_t mNumAvailableUnacquiredBuffers; 261 262 // Number of frames acquired from consumer (debug only) 263 // (as in aquireBuffer called, and release needs to be called) 264 int32_t mNumOutstandingAcquires; 265 266 // Acquire a buffer from the BQ and store it in |item| if successful 267 // \return OK on success, or error on failure. 268 status_t acquireBuffer_l(VideoBuffer *item); 269 270 // Called when a buffer was acquired from the producer 271 void onBufferAcquired_l(const VideoBuffer &buffer); 272 273 // marks the buffer at the slot no longer cached, and accounts for the outstanding 274 // acquire count. Returns true if the slot was populated; otherwise, false. 275 bool discardBufferInSlot_l(slot_id i); 276 277 // marks the buffer at the slot index no longer cached, and accounts for the outstanding 278 // acquire count 279 void discardBufferAtSlotIndex_l(ssize_t bsi); 280 281 // release all acquired and unacquired available buffers 282 // This method will return if it fails to acquire an unacquired available buffer, which will 283 // leave mNumAvailableUnacquiredBuffers positive on return. 284 void releaseAllAvailableBuffers_l(); 285 286 // returns whether we have any available buffers (acquired or not-yet-acquired) 287 bool haveAvailableBuffers_l() const { 288 return !mAvailableBuffers.empty() || mNumAvailableUnacquiredBuffers > 0; 289 } 290 291 // Codec buffers 292 // ------------- 293 294 // When we queue buffers to the encoder, we must hold the references to the graphic buffers 295 // in those buffers - as the producer may free the slots. 296 297 typedef int32_t codec_buffer_id; 298 299 // set of codec buffer ID-s of buffers available to fill 300 List<codec_buffer_id> mFreeCodecBuffers; 301 302 // maps codec buffer ID-s to buffer info submitted to the codec. Used to keep a reference for 303 // the graphics buffer. 304 KeyedVector<codec_buffer_id, std::shared_ptr<AcquiredBuffer>> mSubmittedCodecBuffers; 305 306 // Processes the next acquired frame. If there is no available codec buffer, it returns false 307 // without any further action. 308 // 309 // Otherwise, it consumes the next acquired frame and determines if it needs to be discarded or 310 // dropped. If neither are needed, it submits it to the codec. It also saves the latest 311 // non-dropped frame and submits it for repeat encoding (if this is enabled). 312 // 313 // \require there must be an acquired frame (i.e. we're in the onFrameAvailable callback, 314 // or if we're in codecBufferEmptied and mNumFramesAvailable is nonzero). 315 // \require codec must be executing 316 // \returns true if acquired (and handled) the next frame. Otherwise, false. 317 bool fillCodecBuffer_l(); 318 319 // Calculates the media timestamp for |item| and on success it submits the buffer to the codec, 320 // while also keeping a reference for it in mSubmittedCodecBuffers. 321 // Returns UNKNOWN_ERROR if the buffer was not submitted due to buffer timestamp. Otherwise, 322 // it returns any submit success or error value returned by the codec. 323 status_t submitBuffer_l(const VideoBuffer &item); 324 325 // Submits an empty buffer, with the EOS flag set if there is an available codec buffer and 326 // sets mEndOfStreamSent flag. Does nothing if there is no codec buffer available. 327 void submitEndOfInputStream_l(); 328 329 // Set to true if we want to send end-of-stream after we run out of available frames from the 330 // producer 331 bool mEndOfStream; 332 333 // Flag that the EOS was submitted to the encoder 334 bool mEndOfStreamSent; 335 336 // Dataspace for the last frame submitted to the codec 337 android_dataspace mLastDataspace; 338 339 // Default color aspects for this source 340 int32_t mDefaultColorAspectsPacked; 341 342 // called when the data space of the input buffer changes 343 void onDataspaceChanged_l(android_dataspace dataspace, android_pixel_format pixelFormat); 344 345 // Pointer back to the component that created us. We send buffers here. 346 sp<ComponentWrapper> mComponent; 347 348 // Set by start() / stop(). 349 bool mExecuting; 350 351 bool mSuspended; 352 353 // returns true if this source is unconditionally discarding acquired buffers at the moment 354 // regardless of the metadata of those buffers 355 bool areWeDiscardingAvailableBuffers_l(); 356 357 int64_t mLastFrameTimestampUs; 358 359 // Our BufferQueue interfaces. mProducer is passed to the producer through 360 // getIGraphicBufferProducer, and mConsumer is used internally to retrieve 361 // the buffers queued by the producer. 362 sp<IGraphicBufferProducer> mProducer; 363 sp<IGraphicBufferConsumer> mConsumer; 364 365 // The time to stop sending buffers. 366 int64_t mStopTimeUs; 367 368 struct ActionItem { 369 typedef enum { 370 PAUSE, 371 RESUME, 372 STOP 373 } ActionType; 374 ActionType mAction; 375 int64_t mActionTimeUs; 376 }; 377 378 // Maintain last action timestamp to ensure all the action timestamps are 379 // monotonically increasing. 380 int64_t mLastActionTimeUs; 381 382 // An action queue that queue up all the actions sent to GraphicBufferSource. 383 // STOP action should only show up at the end of the list as all the actions 384 // after a STOP action will be discarded. mActionQueue is protected by mMutex. 385 List<ActionItem> mActionQueue; 386 387 //// 388 friend struct AHandlerReflector<GraphicBufferSource>; 389 390 enum { 391 kWhatRepeatLastFrame, ///< queue last frame for reencoding 392 }; 393 enum { 394 kRepeatLastFrameCount = 10, 395 }; 396 397 int64_t mSkipFramesBeforeNs; 398 399 sp<FrameDropper> mFrameDropper; 400 401 sp<ALooper> mLooper; 402 sp<AHandlerReflector<GraphicBufferSource> > mReflector; 403 404 // Repeat last frame feature 405 // ------------------------- 406 // configuration parameter: repeat interval for frame repeating (<0 if repeating is disabled) 407 int64_t mFrameRepeatIntervalUs; 408 409 // current frame repeat generation - used to cancel a pending frame repeat 410 int32_t mRepeatLastFrameGeneration; 411 412 // number of times to repeat latest frame (0 = none) 413 int32_t mOutstandingFrameRepeatCount; 414 415 // The previous buffer should've been repeated but 416 // no codec buffer was available at the time. 417 bool mFrameRepeatBlockedOnCodecBuffer; 418 419 // hold a reference to the last acquired (and not discarded) frame for frame repeating 420 VideoBuffer mLatestBuffer; 421 422 // queue last frame for reencode after the repeat interval. 423 void queueFrameRepeat_l(); 424 425 // save |item| as the latest buffer and queue it for reencode (repeat) 426 void setLatestBuffer_l(const VideoBuffer &item); 427 428 // submit last frame to encoder and queue it for reencode 429 // \return true if buffer was submitted, false if it wasn't (e.g. source is suspended, there 430 // is no available codec buffer) 431 bool repeatLatestBuffer_l(); 432 433 // Time lapse / slow motion configuration 434 // -------------------------------------- 435 436 // desired frame rate for encoding - value <= 0 if undefined 437 double mFps; 438 439 // desired frame rate for capture - value <= 0 if undefined 440 double mCaptureFps; 441 442 // Time lapse mode is enabled if the capture frame rate is defined and it is 443 // smaller than half the encoding frame rate (if defined). In this mode, 444 // frames that come in between the capture interval (the reciprocal of the 445 // capture frame rate) are dropped and the encoding timestamp is adjusted to 446 // match the desired encoding frame rate. 447 // 448 // Slow motion mode is enabled if both encoding and capture frame rates are 449 // defined and the encoding frame rate is less than half the capture frame 450 // rate. In this mode, the source is expected to produce frames with an even 451 // timestamp interval (after rounding) with the configured capture fps. The 452 // first source timestamp is used as the source base time. Afterwards, the 453 // timestamp of each source frame is snapped to the nearest expected capture 454 // timestamp and scaled to match the configured encoding frame rate. 455 456 // These modes must be enabled before using this source. 457 458 // adjusted capture timestamp of the base frame 459 int64_t mBaseCaptureUs; 460 461 // adjusted encoding timestamp of the base frame 462 int64_t mBaseFrameUs; 463 464 // number of frames from the base time 465 int64_t mFrameCount; 466 467 // adjusted capture timestamp for previous frame (negative if there were 468 // none) 469 int64_t mPrevCaptureUs; 470 471 // adjusted media timestamp for previous frame (negative if there were none) 472 int64_t mPrevFrameUs; 473 474 // desired offset between media time and capture time 475 int64_t mInputBufferTimeOffsetUs; 476 477 // Calculates and outputs the timestamp to use for a buffer with a specific buffer timestamp 478 // |bufferTimestampNs|. Returns false on failure (buffer too close or timestamp is moving 479 // backwards). Otherwise, stores the media timestamp in |*codecTimeUs| and returns true. 480 // 481 // This method takes into account the start time offset and any time lapse or slow motion time 482 // adjustment requests. 483 bool calculateCodecTimestamp_l(nsecs_t bufferTimeNs, int64_t *codecTimeUs); 484 485 void onMessageReceived(const sp<AMessage> &msg); 486 487 DISALLOW_EVIL_CONSTRUCTORS(GraphicBufferSource); 488 }; 489 490 } // namespace android 491 492 #endif // GRAPHIC_BUFFER_SOURCE_H_ 493