1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef MEDIA_BASE_VIDEO_FRAME_H_ 6 #define MEDIA_BASE_VIDEO_FRAME_H_ 7 8 #include <vector> 9 10 #include "base/callback.h" 11 #include "base/md5.h" 12 #include "base/memory/shared_memory.h" 13 #include "base/synchronization/lock.h" 14 #include "media/base/buffers.h" 15 #include "ui/gfx/rect.h" 16 #include "ui/gfx/size.h" 17 18 #if defined(OS_MACOSX) 19 #include <CoreVideo/CVPixelBuffer.h> 20 #include "base/mac/scoped_cftyperef.h" 21 #endif 22 23 class SkBitmap; 24 25 namespace gpu { 26 struct MailboxHolder; 27 } // namespace gpu 28 29 namespace media { 30 31 class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> { 32 public: 33 enum { 34 kFrameSizeAlignment = 16, 35 kFrameSizePadding = 16, 36 kFrameAddressAlignment = 32 37 }; 38 39 enum { 40 kMaxPlanes = 4, 41 42 kYPlane = 0, 43 kUPlane = 1, 44 kUVPlane = kUPlane, 45 kVPlane = 2, 46 kAPlane = 3, 47 }; 48 49 // Surface formats roughly based on FOURCC labels, see: 50 // http://www.fourcc.org/rgb.php 51 // http://www.fourcc.org/yuv.php 52 // Logged to UMA, so never reuse values. 53 enum Format { 54 UNKNOWN = 0, // Unknown format value. 55 YV12 = 1, // 12bpp YVU planar 1x1 Y, 2x2 VU samples 56 YV16 = 2, // 16bpp YVU planar 1x1 Y, 2x1 VU samples 57 I420 = 3, // 12bpp YVU planar 1x1 Y, 2x2 UV samples. 58 YV12A = 4, // 20bpp YUVA planar 1x1 Y, 2x2 VU, 1x1 A samples. 59 #if defined(VIDEO_HOLE) 60 HOLE = 5, // Hole frame. 61 #endif // defined(VIDEO_HOLE) 62 NATIVE_TEXTURE = 6, // Native texture. Pixel-format agnostic. 63 YV12J = 7, // JPEG color range version of YV12 64 NV12 = 8, // 12bpp 1x1 Y plane followed by an interleaved 2x2 UV plane. 65 YV24 = 9, // 24bpp YUV planar, no subsampling. 66 FORMAT_MAX = YV24, // Must always be equal to largest entry logged. 67 }; 68 69 // Returns the name of a Format as a string. 70 static std::string FormatToString(Format format); 71 72 // Creates a new frame in system memory with given parameters. Buffers for 73 // the frame are allocated but not initialized. 74 static scoped_refptr<VideoFrame> CreateFrame( 75 Format format, 76 const gfx::Size& coded_size, 77 const gfx::Rect& visible_rect, 78 const gfx::Size& natural_size, 79 base::TimeDelta timestamp); 80 81 // Call prior to CreateFrame to ensure validity of frame configuration. Called 82 // automatically by VideoDecoderConfig::IsValidConfig(). 83 // TODO(scherkus): VideoDecoderConfig shouldn't call this method 84 static bool IsValidConfig(Format format, const gfx::Size& coded_size, 85 const gfx::Rect& visible_rect, 86 const gfx::Size& natural_size); 87 88 // CB to write pixels from the texture backing this frame into the 89 // |const SkBitmap&| parameter. 90 typedef base::Callback<void(const SkBitmap&)> ReadPixelsCB; 91 92 // CB to be called on the mailbox backing this frame when the frame is 93 // destroyed. 94 typedef base::Callback<void(uint32)> ReleaseMailboxCB; 95 96 // Wraps a native texture of the given parameters with a VideoFrame. The 97 // backing of the VideoFrame is held in the mailbox held by |mailbox_holder|, 98 // and |mailbox_holder_release_cb| will be called with |mailbox_holder| as the 99 // argument when the VideoFrame is to be destroyed. 100 // |read_pixels_cb| may be used to do (slow!) readbacks from the 101 // texture to main memory. 102 static scoped_refptr<VideoFrame> WrapNativeTexture( 103 scoped_ptr<gpu::MailboxHolder> mailbox_holder, 104 const ReleaseMailboxCB& mailbox_holder_release_cb, 105 const gfx::Size& coded_size, 106 const gfx::Rect& visible_rect, 107 const gfx::Size& natural_size, 108 base::TimeDelta timestamp, 109 const ReadPixelsCB& read_pixels_cb); 110 111 #if !defined(MEDIA_FOR_CAST_IOS) 112 // Read pixels from the native texture backing |*this| and write 113 // them to |pixels| as BGRA. |pixels| must point to a buffer at 114 // least as large as 4 * visible_rect().size().GetArea(). 115 void ReadPixelsFromNativeTexture(const SkBitmap& pixels); 116 #endif 117 118 // Wraps packed image data residing in a memory buffer with a VideoFrame. 119 // The image data resides in |data| and is assumed to be packed tightly in a 120 // buffer of logical dimensions |coded_size| with the appropriate bit depth 121 // and plane count as given by |format|. The shared memory handle of the 122 // backing allocation, if present, can be passed in with |handle|. When the 123 // frame is destroyed, |no_longer_needed_cb.Run()| will be called. 124 // Returns NULL on failure. 125 static scoped_refptr<VideoFrame> WrapExternalPackedMemory( 126 Format format, 127 const gfx::Size& coded_size, 128 const gfx::Rect& visible_rect, 129 const gfx::Size& natural_size, 130 uint8* data, 131 size_t data_size, 132 base::SharedMemoryHandle handle, 133 base::TimeDelta timestamp, 134 const base::Closure& no_longer_needed_cb); 135 136 #if defined(OS_POSIX) 137 // Wraps provided dmabufs 138 // (https://www.kernel.org/doc/Documentation/dma-buf-sharing.txt) with a 139 // VideoFrame. The dmabuf fds are dup()ed on creation, so that the VideoFrame 140 // retains a reference to them, and are automatically close()d on destruction, 141 // dropping the reference. The caller may safely close() its reference after 142 // calling WrapExternalDmabufs(). 143 // The image data is only accessible via dmabuf fds, which are usually passed 144 // directly to a hardware device and/or to another process, or can also be 145 // mapped via mmap() for CPU access. 146 // When the frame is destroyed, |no_longer_needed_cb.Run()| will be called. 147 // Returns NULL on failure. 148 static scoped_refptr<VideoFrame> WrapExternalDmabufs( 149 Format format, 150 const gfx::Size& coded_size, 151 const gfx::Rect& visible_rect, 152 const gfx::Size& natural_size, 153 const std::vector<int> dmabuf_fds, 154 base::TimeDelta timestamp, 155 const base::Closure& no_longer_needed_cb); 156 #endif 157 158 #if defined(OS_MACOSX) 159 // Wraps a provided CVPixelBuffer with a VideoFrame. The pixel buffer is 160 // retained for the lifetime of the VideoFrame and released upon destruction. 161 // The image data is only accessible via the pixel buffer, which could be 162 // backed by an IOSurface from another process. All the attributes of the 163 // VideoFrame are derived from the pixel buffer, with the exception of the 164 // timestamp. If information is missing or is incompatible (for example, a 165 // pixel format that has no VideoFrame match), NULL is returned. 166 // http://crbug.com/401308 167 static scoped_refptr<VideoFrame> WrapCVPixelBuffer( 168 CVPixelBufferRef cv_pixel_buffer, 169 base::TimeDelta timestamp); 170 #endif 171 172 // Wraps external YUV data of the given parameters with a VideoFrame. 173 // The returned VideoFrame does not own the data passed in. When the frame 174 // is destroyed |no_longer_needed_cb.Run()| will be called. 175 // TODO(sheu): merge this into WrapExternalSharedMemory(). 176 // http://crbug.com/270217 177 static scoped_refptr<VideoFrame> WrapExternalYuvData( 178 Format format, 179 const gfx::Size& coded_size, 180 const gfx::Rect& visible_rect, 181 const gfx::Size& natural_size, 182 int32 y_stride, 183 int32 u_stride, 184 int32 v_stride, 185 uint8* y_data, 186 uint8* u_data, 187 uint8* v_data, 188 base::TimeDelta timestamp, 189 const base::Closure& no_longer_needed_cb); 190 191 // Wraps |frame| and calls |no_longer_needed_cb| when the wrapper VideoFrame 192 // gets destroyed. |visible_rect| must be a sub rect within 193 // frame->visible_rect(). 194 static scoped_refptr<VideoFrame> WrapVideoFrame( 195 const scoped_refptr<VideoFrame>& frame, 196 const gfx::Rect& visible_rect, 197 const gfx::Size& natural_size, 198 const base::Closure& no_longer_needed_cb); 199 200 // Creates a frame which indicates end-of-stream. 201 static scoped_refptr<VideoFrame> CreateEOSFrame(); 202 203 // Allocates YV12 frame based on |size|, and sets its data to the YUV(y,u,v). 204 static scoped_refptr<VideoFrame> CreateColorFrame( 205 const gfx::Size& size, 206 uint8 y, uint8 u, uint8 v, 207 base::TimeDelta timestamp); 208 209 // Allocates YV12 frame based on |size|, and sets its data to the YUV 210 // equivalent of RGB(0,0,0). 211 static scoped_refptr<VideoFrame> CreateBlackFrame(const gfx::Size& size); 212 213 // Allocates YV12A frame based on |size|, and sets its data to the YUVA 214 // equivalent of RGBA(0,0,0,0). 215 static scoped_refptr<VideoFrame> CreateTransparentFrame( 216 const gfx::Size& size); 217 218 #if defined(VIDEO_HOLE) 219 // Allocates a hole frame. 220 static scoped_refptr<VideoFrame> CreateHoleFrame(const gfx::Size& size); 221 #endif // defined(VIDEO_HOLE) 222 223 static size_t NumPlanes(Format format); 224 225 // Returns the required allocation size for a (tightly packed) frame of the 226 // given coded size and format. 227 static size_t AllocationSize(Format format, const gfx::Size& coded_size); 228 229 // Returns the plane size for a plane of the given coded size and format. 230 static gfx::Size PlaneSize(Format format, 231 size_t plane, 232 const gfx::Size& coded_size); 233 234 // Returns the required allocation size for a (tightly packed) plane of the 235 // given coded size and format. 236 static size_t PlaneAllocationSize(Format format, 237 size_t plane, 238 const gfx::Size& coded_size); 239 240 // Returns horizontal bits per pixel for given |plane| and |format|. 241 static int PlaneHorizontalBitsPerPixel(Format format, size_t plane); 242 243 // Returns the number of bytes per row for the given plane, format, and width. 244 // The width may be aligned to format requirements. 245 static size_t RowBytes(size_t plane, Format format, int width); 246 247 // Returns the number of rows for the given plane, format, and height. 248 // The height may be aligned to format requirements. 249 static size_t Rows(size_t plane, Format format, int height); 250 251 Format format() const { return format_; } 252 253 const gfx::Size& coded_size() const { return coded_size_; } 254 const gfx::Rect& visible_rect() const { return visible_rect_; } 255 const gfx::Size& natural_size() const { return natural_size_; } 256 257 int stride(size_t plane) const; 258 259 // Returns the number of bytes per row and number of rows for a given plane. 260 // 261 // As opposed to stride(), row_bytes() refers to the bytes representing 262 // frame data scanlines (coded_size.width() pixels, without stride padding). 263 int row_bytes(size_t plane) const; 264 int rows(size_t plane) const; 265 266 // Returns pointer to the buffer for a given plane. The memory is owned by 267 // VideoFrame object and must not be freed by the caller. 268 uint8* data(size_t plane) const; 269 270 // Returns the mailbox holder of the native texture wrapped by this frame. 271 // Only valid to call if this is a NATIVE_TEXTURE frame. Before using the 272 // mailbox, the caller must wait for the included sync point. 273 const gpu::MailboxHolder* mailbox_holder() const; 274 275 // Returns the shared-memory handle, if present 276 base::SharedMemoryHandle shared_memory_handle() const; 277 278 #if defined(OS_POSIX) 279 // Returns backing dmabuf file descriptor for given |plane|, if present. 280 int dmabuf_fd(size_t plane) const; 281 #endif 282 283 #if defined(OS_MACOSX) 284 // Returns the backing CVPixelBuffer, if present. 285 CVPixelBufferRef cv_pixel_buffer() const; 286 #endif 287 288 // Returns true if this VideoFrame represents the end of the stream. 289 bool end_of_stream() const { return end_of_stream_; } 290 291 base::TimeDelta timestamp() const { 292 return timestamp_; 293 } 294 void set_timestamp(const base::TimeDelta& timestamp) { 295 timestamp_ = timestamp; 296 } 297 298 class SyncPointClient { 299 public: 300 SyncPointClient() {} 301 virtual uint32 InsertSyncPoint() = 0; 302 virtual void WaitSyncPoint(uint32 sync_point) = 0; 303 304 protected: 305 virtual ~SyncPointClient() {} 306 307 DISALLOW_COPY_AND_ASSIGN(SyncPointClient); 308 }; 309 // It uses |client| to insert a new sync point and potentially waits on a 310 // older sync point. The final sync point will be used to release this 311 // VideoFrame. 312 // This method is thread safe. Both blink and compositor threads can call it. 313 void UpdateReleaseSyncPoint(SyncPointClient* client); 314 315 // Used to keep a running hash of seen frames. Expects an initialized MD5 316 // context. Calls MD5Update with the context and the contents of the frame. 317 void HashFrameForTesting(base::MD5Context* context); 318 319 private: 320 friend class base::RefCountedThreadSafe<VideoFrame>; 321 322 // Returns true if |plane| is a valid plane number for the given format. This 323 // can be used to DCHECK() plane parameters. 324 static bool IsValidPlane(size_t plane, VideoFrame::Format format); 325 326 // Clients must use the static CreateFrame() method to create a new frame. 327 VideoFrame(Format format, 328 const gfx::Size& coded_size, 329 const gfx::Rect& visible_rect, 330 const gfx::Size& natural_size, 331 scoped_ptr<gpu::MailboxHolder> mailbox_holder, 332 base::TimeDelta timestamp, 333 bool end_of_stream); 334 virtual ~VideoFrame(); 335 336 void AllocateYUV(); 337 338 // Frame format. 339 const Format format_; 340 341 // Width and height of the video frame, in pixels. This must include pixel 342 // data for the whole image; i.e. for YUV formats with subsampled chroma 343 // planes, in the case that the visible portion of the image does not line up 344 // on a sample boundary, |coded_size_| must be rounded up appropriately and 345 // the pixel data provided for the odd pixels. 346 const gfx::Size coded_size_; 347 348 // Width, height, and offsets of the visible portion of the video frame. Must 349 // be a subrect of |coded_size_|. Can be odd with respect to the sample 350 // boundaries, e.g. for formats with subsampled chroma. 351 const gfx::Rect visible_rect_; 352 353 // Width and height of the visible portion of the video frame 354 // (|visible_rect_.size()|) with aspect ratio taken into account. 355 const gfx::Size natural_size_; 356 357 // Array of strides for each plane, typically greater or equal to the width 358 // of the surface divided by the horizontal sampling period. Note that 359 // strides can be negative. 360 int32 strides_[kMaxPlanes]; 361 362 // Array of data pointers to each plane. 363 uint8* data_[kMaxPlanes]; 364 365 // Native texture mailbox, if this is a NATIVE_TEXTURE frame. 366 const scoped_ptr<gpu::MailboxHolder> mailbox_holder_; 367 ReleaseMailboxCB mailbox_holder_release_cb_; 368 ReadPixelsCB read_pixels_cb_; 369 370 // Shared memory handle, if this frame was allocated from shared memory. 371 base::SharedMemoryHandle shared_memory_handle_; 372 373 #if defined(OS_POSIX) 374 // Dmabufs for each plane, if this frame is wrapping memory 375 // acquired via dmabuf. 376 base::ScopedFD dmabuf_fds_[kMaxPlanes]; 377 #endif 378 379 #if defined(OS_MACOSX) 380 // CVPixelBuffer, if this frame is wrapping one. 381 base::ScopedCFTypeRef<CVPixelBufferRef> cv_pixel_buffer_; 382 #endif 383 384 base::Closure no_longer_needed_cb_; 385 386 base::TimeDelta timestamp_; 387 388 base::Lock release_sync_point_lock_; 389 uint32 release_sync_point_; 390 391 const bool end_of_stream_; 392 393 DISALLOW_IMPLICIT_CONSTRUCTORS(VideoFrame); 394 }; 395 396 } // namespace media 397 398 #endif // MEDIA_BASE_VIDEO_FRAME_H_ 399
