Home | History | Annotate | Download | only in h264 
      1 /*
      2  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  *
     10  */
     11 
     12 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.h"
     13 
     14 #if defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)
     15 
     16 #include "libyuv/convert.h"
     17 #include "webrtc/base/checks.h"
     18 #include "webrtc/base/logging.h"
     19 #include "webrtc/common_video/include/video_frame_buffer.h"
     20 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h"
     21 #include "webrtc/video_frame.h"
     22 
     23 namespace internal {
     24 
     25 // Convenience function for creating a dictionary.
     26 inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys,
     27                                           CFTypeRef* values,
     28                                           size_t size) {
     29   return CFDictionaryCreate(nullptr, keys, values, size,
     30                             &kCFTypeDictionaryKeyCallBacks,
     31                             &kCFTypeDictionaryValueCallBacks);
     32 }
     33 
     34 // Struct that we pass to the decoder per frame to decode. We receive it again
     35 // in the decoder callback.
     36 struct FrameDecodeParams {
     37   FrameDecodeParams(webrtc::DecodedImageCallback* cb, int64_t ts)
     38       : callback(cb), timestamp(ts) {}
     39   webrtc::DecodedImageCallback* callback;
     40   int64_t timestamp;
     41 };
     42 
     43 // On decode we receive a CVPixelBuffer, which we need to convert to a frame
     44 // buffer for use in the rest of WebRTC. Unfortunately this involves a frame
     45 // copy.
     46 // TODO(tkchin): Stuff CVPixelBuffer into a TextureBuffer and pass that along
     47 // instead once the pipeline supports it.
     48 rtc::scoped_refptr<webrtc::VideoFrameBuffer> VideoFrameBufferForPixelBuffer(
     49     CVPixelBufferRef pixel_buffer) {
     50   RTC_DCHECK(pixel_buffer);
     51   RTC_DCHECK(CVPixelBufferGetPixelFormatType(pixel_buffer) ==
     52              kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
     53   size_t width = CVPixelBufferGetWidthOfPlane(pixel_buffer, 0);
     54   size_t height = CVPixelBufferGetHeightOfPlane(pixel_buffer, 0);
     55   // TODO(tkchin): Use a frame buffer pool.
     56   rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer =
     57       new rtc::RefCountedObject<webrtc::I420Buffer>(width, height);
     58   CVPixelBufferLockBaseAddress(pixel_buffer, kCVPixelBufferLock_ReadOnly);
     59   const uint8_t* src_y = reinterpret_cast<const uint8_t*>(
     60       CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0));
     61   int src_y_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0);
     62   const uint8_t* src_uv = reinterpret_cast<const uint8_t*>(
     63       CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1));
     64   int src_uv_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1);
     65   int ret = libyuv::NV12ToI420(
     66       src_y, src_y_stride, src_uv, src_uv_stride,
     67       buffer->MutableData(webrtc::kYPlane), buffer->stride(webrtc::kYPlane),
     68       buffer->MutableData(webrtc::kUPlane), buffer->stride(webrtc::kUPlane),
     69       buffer->MutableData(webrtc::kVPlane), buffer->stride(webrtc::kVPlane),
     70       width, height);
     71   CVPixelBufferUnlockBaseAddress(pixel_buffer, kCVPixelBufferLock_ReadOnly);
     72   if (ret) {
     73     LOG(LS_ERROR) << "Error converting NV12 to I420: " << ret;
     74     return nullptr;
     75   }
     76   return buffer;
     77 }
     78 
     79 // This is the callback function that VideoToolbox calls when decode is
     80 // complete.
     81 void VTDecompressionOutputCallback(void* decoder,
     82                                    void* params,
     83                                    OSStatus status,
     84                                    VTDecodeInfoFlags info_flags,
     85                                    CVImageBufferRef image_buffer,
     86                                    CMTime timestamp,
     87                                    CMTime duration) {
     88   rtc::scoped_ptr<FrameDecodeParams> decode_params(
     89       reinterpret_cast<FrameDecodeParams*>(params));
     90   if (status != noErr) {
     91     LOG(LS_ERROR) << "Failed to decode frame. Status: " << status;
     92     return;
     93   }
     94   // TODO(tkchin): Handle CVO properly.
     95   rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer =
     96       VideoFrameBufferForPixelBuffer(image_buffer);
     97   webrtc::VideoFrame decoded_frame(buffer, decode_params->timestamp, 0,
     98                                    webrtc::kVideoRotation_0);
     99   decode_params->callback->Decoded(decoded_frame);
    100 }
    101 
    102 }  // namespace internal
    103 
    104 namespace webrtc {
    105 
    106 H264VideoToolboxDecoder::H264VideoToolboxDecoder()
    107     : callback_(nullptr),
    108       video_format_(nullptr),
    109       decompression_session_(nullptr) {}
    110 
    111 H264VideoToolboxDecoder::~H264VideoToolboxDecoder() {
    112   DestroyDecompressionSession();
    113   SetVideoFormat(nullptr);
    114 }
    115 
    116 int H264VideoToolboxDecoder::InitDecode(const VideoCodec* video_codec,
    117                                         int number_of_cores) {
    118   return WEBRTC_VIDEO_CODEC_OK;
    119 }
    120 
    121 int H264VideoToolboxDecoder::Decode(
    122     const EncodedImage& input_image,
    123     bool missing_frames,
    124     const RTPFragmentationHeader* fragmentation,
    125     const CodecSpecificInfo* codec_specific_info,
    126     int64_t render_time_ms) {
    127   RTC_DCHECK(input_image._buffer);
    128 
    129   CMSampleBufferRef sample_buffer = nullptr;
    130   if (!H264AnnexBBufferToCMSampleBuffer(input_image._buffer,
    131                                         input_image._length, video_format_,
    132                                         &sample_buffer)) {
    133     return WEBRTC_VIDEO_CODEC_ERROR;
    134   }
    135   RTC_DCHECK(sample_buffer);
    136   // Check if the video format has changed, and reinitialize decoder if needed.
    137   CMVideoFormatDescriptionRef description =
    138       CMSampleBufferGetFormatDescription(sample_buffer);
    139   if (!CMFormatDescriptionEqual(description, video_format_)) {
    140     SetVideoFormat(description);
    141     ResetDecompressionSession();
    142   }
    143   VTDecodeFrameFlags decode_flags =
    144       kVTDecodeFrame_EnableAsynchronousDecompression;
    145   rtc::scoped_ptr<internal::FrameDecodeParams> frame_decode_params;
    146   frame_decode_params.reset(
    147       new internal::FrameDecodeParams(callback_, input_image._timeStamp));
    148   OSStatus status = VTDecompressionSessionDecodeFrame(
    149       decompression_session_, sample_buffer, decode_flags,
    150       frame_decode_params.release(), nullptr);
    151   CFRelease(sample_buffer);
    152   if (status != noErr) {
    153     LOG(LS_ERROR) << "Failed to decode frame with code: " << status;
    154     return WEBRTC_VIDEO_CODEC_ERROR;
    155   }
    156   return WEBRTC_VIDEO_CODEC_OK;
    157 }
    158 
    159 int H264VideoToolboxDecoder::RegisterDecodeCompleteCallback(
    160     DecodedImageCallback* callback) {
    161   RTC_DCHECK(!callback_);
    162   callback_ = callback;
    163   return WEBRTC_VIDEO_CODEC_OK;
    164 }
    165 
    166 int H264VideoToolboxDecoder::Release() {
    167   callback_ = nullptr;
    168   return WEBRTC_VIDEO_CODEC_OK;
    169 }
    170 
    171 int H264VideoToolboxDecoder::Reset() {
    172   ResetDecompressionSession();
    173   return WEBRTC_VIDEO_CODEC_OK;
    174 }
    175 
    176 int H264VideoToolboxDecoder::ResetDecompressionSession() {
    177   DestroyDecompressionSession();
    178 
    179   // Need to wait for the first SPS to initialize decoder.
    180   if (!video_format_) {
    181     return WEBRTC_VIDEO_CODEC_OK;
    182   }
    183 
    184   // Set keys for OpenGL and IOSurface compatibilty, which makes the encoder
    185   // create pixel buffers with GPU backed memory. The intent here is to pass
    186   // the pixel buffers directly so we avoid a texture upload later during
    187   // rendering. This currently is moot because we are converting back to an
    188   // I420 frame after decode, but eventually we will be able to plumb
    189   // CVPixelBuffers directly to the renderer.
    190   // TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that
    191   // we can pass CVPixelBuffers as native handles in decoder output.
    192   static size_t const attributes_size = 3;
    193   CFTypeRef keys[attributes_size] = {
    194 #if defined(WEBRTC_IOS)
    195     kCVPixelBufferOpenGLESCompatibilityKey,
    196 #elif defined(WEBRTC_MAC)
    197     kCVPixelBufferOpenGLCompatibilityKey,
    198 #endif
    199     kCVPixelBufferIOSurfacePropertiesKey,
    200     kCVPixelBufferPixelFormatTypeKey
    201   };
    202   CFDictionaryRef io_surface_value =
    203       internal::CreateCFDictionary(nullptr, nullptr, 0);
    204   int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
    205   CFNumberRef pixel_format =
    206       CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
    207   CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value,
    208                                        pixel_format};
    209   CFDictionaryRef attributes =
    210       internal::CreateCFDictionary(keys, values, attributes_size);
    211   if (io_surface_value) {
    212     CFRelease(io_surface_value);
    213     io_surface_value = nullptr;
    214   }
    215   if (pixel_format) {
    216     CFRelease(pixel_format);
    217     pixel_format = nullptr;
    218   }
    219   VTDecompressionOutputCallbackRecord record = {
    220       internal::VTDecompressionOutputCallback, this,
    221   };
    222   OSStatus status =
    223       VTDecompressionSessionCreate(nullptr, video_format_, nullptr, attributes,
    224                                    &record, &decompression_session_);
    225   CFRelease(attributes);
    226   if (status != noErr) {
    227     DestroyDecompressionSession();
    228     return WEBRTC_VIDEO_CODEC_ERROR;
    229   }
    230   ConfigureDecompressionSession();
    231 
    232   return WEBRTC_VIDEO_CODEC_OK;
    233 }
    234 
    235 void H264VideoToolboxDecoder::ConfigureDecompressionSession() {
    236   RTC_DCHECK(decompression_session_);
    237 #if defined(WEBRTC_IOS)
    238   VTSessionSetProperty(decompression_session_,
    239                        kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
    240 #endif
    241 }
    242 
    243 void H264VideoToolboxDecoder::DestroyDecompressionSession() {
    244   if (decompression_session_) {
    245     VTDecompressionSessionInvalidate(decompression_session_);
    246     decompression_session_ = nullptr;
    247   }
    248 }
    249 
    250 void H264VideoToolboxDecoder::SetVideoFormat(
    251     CMVideoFormatDescriptionRef video_format) {
    252   if (video_format_ == video_format) {
    253     return;
    254   }
    255   if (video_format_) {
    256     CFRelease(video_format_);
    257   }
    258   video_format_ = video_format;
    259   if (video_format_) {
    260     CFRetain(video_format_);
    261   }
    262 }
    263 
    264 const char* H264VideoToolboxDecoder::ImplementationName() const {
    265   return "VideoToolbox";
    266 }
    267 
    268 }  // namespace webrtc
    269 
    270 #endif  // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)
    271