Home | History | Annotate | Download | only in filters 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "media/filters/ffmpeg_video_decoder.h"
      6 
      7 #include <algorithm>
      8 #include <string>
      9 
     10 #include "base/bind.h"
     11 #include "base/callback_helpers.h"
     12 #include "base/command_line.h"
     13 #include "base/location.h"
     14 #include "base/message_loop/message_loop_proxy.h"
     15 #include "base/strings/string_number_conversions.h"
     16 #include "media/base/bind_to_loop.h"
     17 #include "media/base/decoder_buffer.h"
     18 #include "media/base/limits.h"
     19 #include "media/base/media_switches.h"
     20 #include "media/base/pipeline.h"
     21 #include "media/base/video_decoder_config.h"
     22 #include "media/base/video_frame.h"
     23 #include "media/base/video_util.h"
     24 #include "media/ffmpeg/ffmpeg_common.h"
     25 #include "media/filters/ffmpeg_glue.h"
     26 
     27 namespace media {
     28 
     29 // Always try to use three threads for video decoding.  There is little reason
     30 // not to since current day CPUs tend to be multi-core and we measured
     31 // performance benefits on older machines such as P4s with hyperthreading.
     32 //
     33 // Handling decoding on separate threads also frees up the pipeline thread to
     34 // continue processing. Although it'd be nice to have the option of a single
     35 // decoding thread, FFmpeg treats having one thread the same as having zero
     36 // threads (i.e., avcodec_decode_video() will execute on the calling thread).
     37 // Yet another reason for having two threads :)
     38 static const int kDecodeThreads = 2;
     39 static const int kMaxDecodeThreads = 16;
     40 
     41 // Returns the number of threads given the FFmpeg CodecID. Also inspects the
     42 // command line for a valid --video-threads flag.
     43 static int GetThreadCount(AVCodecID codec_id) {
     44   // Refer to http://crbug.com/93932 for tsan suppressions on decoding.
     45   int decode_threads = kDecodeThreads;
     46 
     47   const CommandLine* cmd_line = CommandLine::ForCurrentProcess();
     48   std::string threads(cmd_line->GetSwitchValueASCII(switches::kVideoThreads));
     49   if (threads.empty() || !base::StringToInt(threads, &decode_threads))
     50     return decode_threads;
     51 
     52   decode_threads = std::max(decode_threads, 0);
     53   decode_threads = std::min(decode_threads, kMaxDecodeThreads);
     54   return decode_threads;
     55 }
     56 
     57 FFmpegVideoDecoder::FFmpegVideoDecoder(
     58     const scoped_refptr<base::MessageLoopProxy>& message_loop)
     59     : message_loop_(message_loop),
     60       weak_factory_(this),
     61       state_(kUninitialized),
     62       codec_context_(NULL),
     63       av_frame_(NULL) {
     64 }
     65 
     66 int FFmpegVideoDecoder::GetVideoBuffer(AVCodecContext* codec_context,
     67                                        AVFrame* frame) {
     68   // Don't use |codec_context_| here! With threaded decoding,
     69   // it will contain unsynchronized width/height/pix_fmt values,
     70   // whereas |codec_context| contains the current threads's
     71   // updated width/height/pix_fmt, which can change for adaptive
     72   // content.
     73   VideoFrame::Format format = PixelFormatToVideoFormat(codec_context->pix_fmt);
     74   if (format == VideoFrame::INVALID)
     75     return AVERROR(EINVAL);
     76   DCHECK(format == VideoFrame::YV12 || format == VideoFrame::YV16);
     77 
     78   gfx::Size size(codec_context->width, codec_context->height);
     79   int ret;
     80   if ((ret = av_image_check_size(size.width(), size.height(), 0, NULL)) < 0)
     81     return ret;
     82 
     83   gfx::Size natural_size;
     84   if (codec_context->sample_aspect_ratio.num > 0) {
     85     natural_size = GetNaturalSize(size,
     86                                   codec_context->sample_aspect_ratio.num,
     87                                   codec_context->sample_aspect_ratio.den);
     88   } else {
     89     natural_size = config_.natural_size();
     90   }
     91 
     92   if (!VideoFrame::IsValidConfig(format, size, gfx::Rect(size), natural_size))
     93     return AVERROR(EINVAL);
     94 
     95   scoped_refptr<VideoFrame> video_frame =
     96       VideoFrame::CreateFrame(format, size, gfx::Rect(size), natural_size,
     97                               kNoTimestamp());
     98 
     99   for (int i = 0; i < 3; i++) {
    100     frame->base[i] = video_frame->data(i);
    101     frame->data[i] = video_frame->data(i);
    102     frame->linesize[i] = video_frame->stride(i);
    103   }
    104 
    105   frame->opaque = NULL;
    106   video_frame.swap(reinterpret_cast<VideoFrame**>(&frame->opaque));
    107   frame->type = FF_BUFFER_TYPE_USER;
    108   frame->pkt_pts = codec_context->pkt ? codec_context->pkt->pts :
    109                                         AV_NOPTS_VALUE;
    110   frame->width = codec_context->width;
    111   frame->height = codec_context->height;
    112   frame->format = codec_context->pix_fmt;
    113 
    114   return 0;
    115 }
    116 
    117 static int GetVideoBufferImpl(AVCodecContext* s, AVFrame* frame) {
    118   FFmpegVideoDecoder* decoder = static_cast<FFmpegVideoDecoder*>(s->opaque);
    119   return decoder->GetVideoBuffer(s, frame);
    120 }
    121 
    122 static void ReleaseVideoBufferImpl(AVCodecContext* s, AVFrame* frame) {
    123   scoped_refptr<VideoFrame> video_frame;
    124   video_frame.swap(reinterpret_cast<VideoFrame**>(&frame->opaque));
    125 
    126   // The FFmpeg API expects us to zero the data pointers in
    127   // this callback
    128   memset(frame->data, 0, sizeof(frame->data));
    129   frame->opaque = NULL;
    130 }
    131 
    132 void FFmpegVideoDecoder::Initialize(const VideoDecoderConfig& config,
    133                                     const PipelineStatusCB& status_cb) {
    134   DCHECK(message_loop_->BelongsToCurrentThread());
    135   DCHECK(decode_cb_.is_null());
    136   DCHECK(reset_cb_.is_null());
    137   DCHECK(!config.is_encrypted());
    138 
    139   FFmpegGlue::InitializeFFmpeg();
    140   weak_this_ = weak_factory_.GetWeakPtr();
    141 
    142   config_ = config;
    143   PipelineStatusCB initialize_cb = BindToCurrentLoop(status_cb);
    144 
    145   if (!config.IsValidConfig() || !ConfigureDecoder()) {
    146     initialize_cb.Run(DECODER_ERROR_NOT_SUPPORTED);
    147     return;
    148   }
    149 
    150   // Success!
    151   state_ = kNormal;
    152   initialize_cb.Run(PIPELINE_OK);
    153 }
    154 
    155 void FFmpegVideoDecoder::Decode(const scoped_refptr<DecoderBuffer>& buffer,
    156                                 const DecodeCB& decode_cb) {
    157   DCHECK(message_loop_->BelongsToCurrentThread());
    158   DCHECK(!decode_cb.is_null());
    159   CHECK_NE(state_, kUninitialized);
    160   CHECK(decode_cb_.is_null()) << "Overlapping decodes are not supported.";
    161   decode_cb_ = BindToCurrentLoop(decode_cb);
    162 
    163   if (state_ == kError) {
    164     base::ResetAndReturn(&decode_cb_).Run(kDecodeError, NULL);
    165     return;
    166   }
    167 
    168   // Return empty frames if decoding has finished.
    169   if (state_ == kDecodeFinished) {
    170     base::ResetAndReturn(&decode_cb_).Run(kOk, VideoFrame::CreateEmptyFrame());
    171     return;
    172   }
    173 
    174   DecodeBuffer(buffer);
    175 }
    176 
    177 void FFmpegVideoDecoder::Reset(const base::Closure& closure) {
    178   DCHECK(message_loop_->BelongsToCurrentThread());
    179   DCHECK(reset_cb_.is_null());
    180   reset_cb_ = BindToCurrentLoop(closure);
    181 
    182   // Defer the reset if a decode is pending.
    183   if (!decode_cb_.is_null())
    184     return;
    185 
    186   DoReset();
    187 }
    188 
    189 void FFmpegVideoDecoder::DoReset() {
    190   DCHECK(decode_cb_.is_null());
    191 
    192   avcodec_flush_buffers(codec_context_);
    193   state_ = kNormal;
    194   base::ResetAndReturn(&reset_cb_).Run();
    195 }
    196 
    197 void FFmpegVideoDecoder::Stop(const base::Closure& closure) {
    198   DCHECK(message_loop_->BelongsToCurrentThread());
    199   base::ScopedClosureRunner runner(BindToCurrentLoop(closure));
    200 
    201   if (state_ == kUninitialized)
    202     return;
    203 
    204   if (!decode_cb_.is_null()) {
    205     base::ResetAndReturn(&decode_cb_).Run(kOk, NULL);
    206     // Reset is pending only when decode is pending.
    207     if (!reset_cb_.is_null())
    208       base::ResetAndReturn(&reset_cb_).Run();
    209   }
    210 
    211   ReleaseFFmpegResources();
    212   state_ = kUninitialized;
    213 }
    214 
    215 FFmpegVideoDecoder::~FFmpegVideoDecoder() {
    216   DCHECK_EQ(kUninitialized, state_);
    217   DCHECK(!codec_context_);
    218   DCHECK(!av_frame_);
    219 }
    220 
    221 void FFmpegVideoDecoder::DecodeBuffer(
    222     const scoped_refptr<DecoderBuffer>& buffer) {
    223   DCHECK(message_loop_->BelongsToCurrentThread());
    224   DCHECK_NE(state_, kUninitialized);
    225   DCHECK_NE(state_, kDecodeFinished);
    226   DCHECK_NE(state_, kError);
    227   DCHECK(reset_cb_.is_null());
    228   DCHECK(!decode_cb_.is_null());
    229   DCHECK(buffer);
    230 
    231   // During decode, because reads are issued asynchronously, it is possible to
    232   // receive multiple end of stream buffers since each decode is acked. When the
    233   // first end of stream buffer is read, FFmpeg may still have frames queued
    234   // up in the decoder so we need to go through the decode loop until it stops
    235   // giving sensible data.  After that, the decoder should output empty
    236   // frames.  There are three states the decoder can be in:
    237   //
    238   //   kNormal: This is the starting state. Buffers are decoded. Decode errors
    239   //            are discarded.
    240   //   kFlushCodec: There isn't any more input data. Call avcodec_decode_video2
    241   //                until no more data is returned to flush out remaining
    242   //                frames. The input buffer is ignored at this point.
    243   //   kDecodeFinished: All calls return empty frames.
    244   //   kError: Unexpected error happened.
    245   //
    246   // These are the possible state transitions.
    247   //
    248   // kNormal -> kFlushCodec:
    249   //     When buffer->end_of_stream() is first true.
    250   // kNormal -> kError:
    251   //     A decoding error occurs and decoding needs to stop.
    252   // kFlushCodec -> kDecodeFinished:
    253   //     When avcodec_decode_video2() returns 0 data.
    254   // kFlushCodec -> kError:
    255   //     When avcodec_decode_video2() errors out.
    256   // (any state) -> kNormal:
    257   //     Any time Reset() is called.
    258 
    259   // Transition to kFlushCodec on the first end of stream buffer.
    260   if (state_ == kNormal && buffer->end_of_stream()) {
    261     state_ = kFlushCodec;
    262   }
    263 
    264   scoped_refptr<VideoFrame> video_frame;
    265   if (!FFmpegDecode(buffer, &video_frame)) {
    266     state_ = kError;
    267     base::ResetAndReturn(&decode_cb_).Run(kDecodeError, NULL);
    268     return;
    269   }
    270 
    271   if (!video_frame.get()) {
    272     if (state_ == kFlushCodec) {
    273       DCHECK(buffer->end_of_stream());
    274       state_ = kDecodeFinished;
    275       base::ResetAndReturn(&decode_cb_)
    276           .Run(kOk, VideoFrame::CreateEmptyFrame());
    277       return;
    278     }
    279 
    280     base::ResetAndReturn(&decode_cb_).Run(kNotEnoughData, NULL);
    281     return;
    282   }
    283 
    284   base::ResetAndReturn(&decode_cb_).Run(kOk, video_frame);
    285 }
    286 
    287 bool FFmpegVideoDecoder::FFmpegDecode(
    288     const scoped_refptr<DecoderBuffer>& buffer,
    289     scoped_refptr<VideoFrame>* video_frame) {
    290   DCHECK(video_frame);
    291 
    292   // Reset frame to default values.
    293   avcodec_get_frame_defaults(av_frame_);
    294 
    295   // Create a packet for input data.
    296   // Due to FFmpeg API changes we no longer have const read-only pointers.
    297   AVPacket packet;
    298   av_init_packet(&packet);
    299   if (buffer->end_of_stream()) {
    300     packet.data = NULL;
    301     packet.size = 0;
    302   } else {
    303     packet.data = const_cast<uint8*>(buffer->data());
    304     packet.size = buffer->data_size();
    305 
    306     // Let FFmpeg handle presentation timestamp reordering.
    307     codec_context_->reordered_opaque = buffer->timestamp().InMicroseconds();
    308 
    309     // This is for codecs not using get_buffer to initialize
    310     // |av_frame_->reordered_opaque|
    311     av_frame_->reordered_opaque = codec_context_->reordered_opaque;
    312   }
    313 
    314   int frame_decoded = 0;
    315   int result = avcodec_decode_video2(codec_context_,
    316                                      av_frame_,
    317                                      &frame_decoded,
    318                                      &packet);
    319   // Log the problem if we can't decode a video frame and exit early.
    320   if (result < 0) {
    321     LOG(ERROR) << "Error decoding video: " << buffer->AsHumanReadableString();
    322     *video_frame = NULL;
    323     return false;
    324   }
    325 
    326   // If no frame was produced then signal that more data is required to
    327   // produce more frames. This can happen under two circumstances:
    328   //   1) Decoder was recently initialized/flushed
    329   //   2) End of stream was reached and all internal frames have been output
    330   if (frame_decoded == 0) {
    331     *video_frame = NULL;
    332     return true;
    333   }
    334 
    335   // TODO(fbarchard): Work around for FFmpeg http://crbug.com/27675
    336   // The decoder is in a bad state and not decoding correctly.
    337   // Checking for NULL avoids a crash in CopyPlane().
    338   if (!av_frame_->data[VideoFrame::kYPlane] ||
    339       !av_frame_->data[VideoFrame::kUPlane] ||
    340       !av_frame_->data[VideoFrame::kVPlane]) {
    341     LOG(ERROR) << "Video frame was produced yet has invalid frame data.";
    342     *video_frame = NULL;
    343     return false;
    344   }
    345 
    346   if (!av_frame_->opaque) {
    347     LOG(ERROR) << "VideoFrame object associated with frame data not set.";
    348     return false;
    349   }
    350   *video_frame = static_cast<VideoFrame*>(av_frame_->opaque);
    351 
    352   (*video_frame)->SetTimestamp(
    353       base::TimeDelta::FromMicroseconds(av_frame_->reordered_opaque));
    354 
    355   return true;
    356 }
    357 
    358 void FFmpegVideoDecoder::ReleaseFFmpegResources() {
    359   if (codec_context_) {
    360     av_free(codec_context_->extradata);
    361     avcodec_close(codec_context_);
    362     av_free(codec_context_);
    363     codec_context_ = NULL;
    364   }
    365   if (av_frame_) {
    366     av_free(av_frame_);
    367     av_frame_ = NULL;
    368   }
    369 }
    370 
    371 bool FFmpegVideoDecoder::ConfigureDecoder() {
    372   // Release existing decoder resources if necessary.
    373   ReleaseFFmpegResources();
    374 
    375   // Initialize AVCodecContext structure.
    376   codec_context_ = avcodec_alloc_context3(NULL);
    377   VideoDecoderConfigToAVCodecContext(config_, codec_context_);
    378 
    379   // Enable motion vector search (potentially slow), strong deblocking filter
    380   // for damaged macroblocks, and set our error detection sensitivity.
    381   codec_context_->error_concealment = FF_EC_GUESS_MVS | FF_EC_DEBLOCK;
    382   codec_context_->thread_count = GetThreadCount(codec_context_->codec_id);
    383   codec_context_->opaque = this;
    384   codec_context_->flags |= CODEC_FLAG_EMU_EDGE;
    385   codec_context_->get_buffer = GetVideoBufferImpl;
    386   codec_context_->release_buffer = ReleaseVideoBufferImpl;
    387 
    388   AVCodec* codec = avcodec_find_decoder(codec_context_->codec_id);
    389   if (!codec || avcodec_open2(codec_context_, codec, NULL) < 0) {
    390     ReleaseFFmpegResources();
    391     return false;
    392   }
    393 
    394   av_frame_ = avcodec_alloc_frame();
    395   return true;
    396 }
    397 
    398 }  // namespace media
    399