Home | History | Annotate | Download | only in filters 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "media/filters/ffmpeg_audio_decoder.h"
      6 
      7 #include "base/callback_helpers.h"
      8 #include "base/single_thread_task_runner.h"
      9 #include "media/base/audio_buffer.h"
     10 #include "media/base/audio_bus.h"
     11 #include "media/base/audio_decoder_config.h"
     12 #include "media/base/audio_discard_helper.h"
     13 #include "media/base/bind_to_current_loop.h"
     14 #include "media/base/decoder_buffer.h"
     15 #include "media/base/limits.h"
     16 #include "media/base/sample_format.h"
     17 #include "media/ffmpeg/ffmpeg_common.h"
     18 #include "media/filters/ffmpeg_glue.h"
     19 
     20 namespace media {
     21 
     22 // Returns true if the decode result was end of stream.
     23 static inline bool IsEndOfStream(int result,
     24                                  int decoded_size,
     25                                  const scoped_refptr<DecoderBuffer>& input) {
     26   // Three conditions to meet to declare end of stream for this decoder:
     27   // 1. FFmpeg didn't read anything.
     28   // 2. FFmpeg didn't output anything.
     29   // 3. An end of stream buffer is received.
     30   return result == 0 && decoded_size == 0 && input->end_of_stream();
     31 }
     32 
     33 // Return the number of channels from the data in |frame|.
     34 static inline int DetermineChannels(AVFrame* frame) {
     35 #if defined(CHROMIUM_NO_AVFRAME_CHANNELS)
     36   // When use_system_ffmpeg==1, libav's AVFrame doesn't have channels field.
     37   return av_get_channel_layout_nb_channels(frame->channel_layout);
     38 #else
     39   return frame->channels;
     40 #endif
     41 }
     42 
     43 // Called by FFmpeg's allocation routine to free a buffer. |opaque| is the
     44 // AudioBuffer allocated, so unref it.
     45 static void ReleaseAudioBufferImpl(void* opaque, uint8* data) {
     46   scoped_refptr<AudioBuffer> buffer;
     47   buffer.swap(reinterpret_cast<AudioBuffer**>(&opaque));
     48 }
     49 
     50 // Called by FFmpeg's allocation routine to allocate a buffer. Uses
     51 // AVCodecContext.opaque to get the object reference in order to call
     52 // GetAudioBuffer() to do the actual allocation.
     53 static int GetAudioBuffer(struct AVCodecContext* s, AVFrame* frame, int flags) {
     54   DCHECK(s->codec->capabilities & CODEC_CAP_DR1);
     55   DCHECK_EQ(s->codec_type, AVMEDIA_TYPE_AUDIO);
     56 
     57   // Since this routine is called by FFmpeg when a buffer is required for audio
     58   // data, use the values supplied by FFmpeg (ignoring the current settings).
     59   // FFmpegDecode() gets to determine if the buffer is useable or not.
     60   AVSampleFormat format = static_cast<AVSampleFormat>(frame->format);
     61   SampleFormat sample_format = AVSampleFormatToSampleFormat(format);
     62   int channels = DetermineChannels(frame);
     63   if (channels <= 0 || channels >= limits::kMaxChannels) {
     64     DLOG(ERROR) << "Requested number of channels (" << channels
     65                 << ") exceeds limit.";
     66     return AVERROR(EINVAL);
     67   }
     68 
     69   int bytes_per_channel = SampleFormatToBytesPerChannel(sample_format);
     70   if (frame->nb_samples <= 0)
     71     return AVERROR(EINVAL);
     72 
     73   if (s->channels != channels) {
     74     DLOG(ERROR) << "AVCodecContext and AVFrame disagree on channel count.";
     75     return AVERROR(EINVAL);
     76   }
     77 
     78   // Determine how big the buffer should be and allocate it. FFmpeg may adjust
     79   // how big each channel data is in order to meet the alignment policy, so
     80   // we need to take this into consideration.
     81   int buffer_size_in_bytes =
     82       av_samples_get_buffer_size(&frame->linesize[0],
     83                                  channels,
     84                                  frame->nb_samples,
     85                                  format,
     86                                  AudioBuffer::kChannelAlignment);
     87   // Check for errors from av_samples_get_buffer_size().
     88   if (buffer_size_in_bytes < 0)
     89     return buffer_size_in_bytes;
     90   int frames_required = buffer_size_in_bytes / bytes_per_channel / channels;
     91   DCHECK_GE(frames_required, frame->nb_samples);
     92   scoped_refptr<AudioBuffer> buffer = AudioBuffer::CreateBuffer(
     93       sample_format,
     94       ChannelLayoutToChromeChannelLayout(s->channel_layout, s->channels),
     95       channels,
     96       s->sample_rate,
     97       frames_required);
     98 
     99   // Initialize the data[] and extended_data[] fields to point into the memory
    100   // allocated for AudioBuffer. |number_of_planes| will be 1 for interleaved
    101   // audio and equal to |channels| for planar audio.
    102   int number_of_planes = buffer->channel_data().size();
    103   if (number_of_planes <= AV_NUM_DATA_POINTERS) {
    104     DCHECK_EQ(frame->extended_data, frame->data);
    105     for (int i = 0; i < number_of_planes; ++i)
    106       frame->data[i] = buffer->channel_data()[i];
    107   } else {
    108     // There are more channels than can fit into data[], so allocate
    109     // extended_data[] and fill appropriately.
    110     frame->extended_data = static_cast<uint8**>(
    111         av_malloc(number_of_planes * sizeof(*frame->extended_data)));
    112     int i = 0;
    113     for (; i < AV_NUM_DATA_POINTERS; ++i)
    114       frame->extended_data[i] = frame->data[i] = buffer->channel_data()[i];
    115     for (; i < number_of_planes; ++i)
    116       frame->extended_data[i] = buffer->channel_data()[i];
    117   }
    118 
    119   // Now create an AVBufferRef for the data just allocated. It will own the
    120   // reference to the AudioBuffer object.
    121   void* opaque = NULL;
    122   buffer.swap(reinterpret_cast<AudioBuffer**>(&opaque));
    123   frame->buf[0] = av_buffer_create(
    124       frame->data[0], buffer_size_in_bytes, ReleaseAudioBufferImpl, opaque, 0);
    125   return 0;
    126 }
    127 
    128 FFmpegAudioDecoder::FFmpegAudioDecoder(
    129     const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
    130     const LogCB& log_cb)
    131     : task_runner_(task_runner),
    132       state_(kUninitialized),
    133       av_sample_format_(0),
    134       log_cb_(log_cb) {
    135 }
    136 
    137 FFmpegAudioDecoder::~FFmpegAudioDecoder() {
    138   DCHECK(task_runner_->BelongsToCurrentThread());
    139 
    140   if (state_ != kUninitialized) {
    141     ReleaseFFmpegResources();
    142     ResetTimestampState();
    143   }
    144 }
    145 
    146 std::string FFmpegAudioDecoder::GetDisplayName() const {
    147   return "FFmpegAudioDecoder";
    148 }
    149 
    150 void FFmpegAudioDecoder::Initialize(const AudioDecoderConfig& config,
    151                                     const PipelineStatusCB& status_cb,
    152                                     const OutputCB& output_cb) {
    153   DCHECK(task_runner_->BelongsToCurrentThread());
    154   DCHECK(!config.is_encrypted());
    155 
    156   FFmpegGlue::InitializeFFmpeg();
    157 
    158   config_ = config;
    159   PipelineStatusCB initialize_cb = BindToCurrentLoop(status_cb);
    160 
    161   if (!config.IsValidConfig() || !ConfigureDecoder()) {
    162     initialize_cb.Run(DECODER_ERROR_NOT_SUPPORTED);
    163     return;
    164   }
    165 
    166   // Success!
    167   output_cb_ = BindToCurrentLoop(output_cb);
    168   state_ = kNormal;
    169   initialize_cb.Run(PIPELINE_OK);
    170 }
    171 
    172 void FFmpegAudioDecoder::Decode(const scoped_refptr<DecoderBuffer>& buffer,
    173                                 const DecodeCB& decode_cb) {
    174   DCHECK(task_runner_->BelongsToCurrentThread());
    175   DCHECK(!decode_cb.is_null());
    176   CHECK_NE(state_, kUninitialized);
    177   DecodeCB decode_cb_bound = BindToCurrentLoop(decode_cb);
    178 
    179   if (state_ == kError) {
    180     decode_cb_bound.Run(kDecodeError);
    181     return;
    182   }
    183 
    184   // Do nothing if decoding has finished.
    185   if (state_ == kDecodeFinished) {
    186     decode_cb_bound.Run(kOk);
    187     return;
    188   }
    189 
    190   DecodeBuffer(buffer, decode_cb_bound);
    191 }
    192 
    193 void FFmpegAudioDecoder::Reset(const base::Closure& closure) {
    194   DCHECK(task_runner_->BelongsToCurrentThread());
    195 
    196   avcodec_flush_buffers(codec_context_.get());
    197   state_ = kNormal;
    198   ResetTimestampState();
    199   task_runner_->PostTask(FROM_HERE, closure);
    200 }
    201 
    202 void FFmpegAudioDecoder::DecodeBuffer(
    203     const scoped_refptr<DecoderBuffer>& buffer,
    204     const DecodeCB& decode_cb) {
    205   DCHECK(task_runner_->BelongsToCurrentThread());
    206   DCHECK_NE(state_, kUninitialized);
    207   DCHECK_NE(state_, kDecodeFinished);
    208   DCHECK_NE(state_, kError);
    209   DCHECK(buffer.get());
    210 
    211   // Make sure we are notified if http://crbug.com/49709 returns.  Issue also
    212   // occurs with some damaged files.
    213   if (!buffer->end_of_stream() && buffer->timestamp() == kNoTimestamp()) {
    214     DVLOG(1) << "Received a buffer without timestamps!";
    215     decode_cb.Run(kDecodeError);
    216     return;
    217   }
    218 
    219   bool has_produced_frame;
    220   do {
    221     has_produced_frame = false;
    222     if (!FFmpegDecode(buffer, &has_produced_frame)) {
    223       state_ = kError;
    224       decode_cb.Run(kDecodeError);
    225       return;
    226     }
    227     // Repeat to flush the decoder after receiving EOS buffer.
    228   } while (buffer->end_of_stream() && has_produced_frame);
    229 
    230   if (buffer->end_of_stream())
    231     state_ = kDecodeFinished;
    232 
    233   decode_cb.Run(kOk);
    234 }
    235 
    236 bool FFmpegAudioDecoder::FFmpegDecode(
    237     const scoped_refptr<DecoderBuffer>& buffer,
    238     bool* has_produced_frame) {
    239   DCHECK(!*has_produced_frame);
    240 
    241   AVPacket packet;
    242   av_init_packet(&packet);
    243   if (buffer->end_of_stream()) {
    244     packet.data = NULL;
    245     packet.size = 0;
    246   } else {
    247     packet.data = const_cast<uint8*>(buffer->data());
    248     packet.size = buffer->data_size();
    249   }
    250 
    251   // Each audio packet may contain several frames, so we must call the decoder
    252   // until we've exhausted the packet.  Regardless of the packet size we always
    253   // want to hand it to the decoder at least once, otherwise we would end up
    254   // skipping end of stream packets since they have a size of zero.
    255   do {
    256     int frame_decoded = 0;
    257     const int result = avcodec_decode_audio4(
    258         codec_context_.get(), av_frame_.get(), &frame_decoded, &packet);
    259 
    260     if (result < 0) {
    261       DCHECK(!buffer->end_of_stream())
    262           << "End of stream buffer produced an error! "
    263           << "This is quite possibly a bug in the audio decoder not handling "
    264           << "end of stream AVPackets correctly.";
    265 
    266       MEDIA_LOG(log_cb_)
    267           << "Dropping audio frame which failed decode with timestamp: "
    268           << buffer->timestamp().InMicroseconds() << " us, duration: "
    269           << buffer->duration().InMicroseconds() << " us, packet size: "
    270           << buffer->data_size() << " bytes";
    271 
    272       break;
    273     }
    274 
    275     // Update packet size and data pointer in case we need to call the decoder
    276     // with the remaining bytes from this packet.
    277     packet.size -= result;
    278     packet.data += result;
    279 
    280     scoped_refptr<AudioBuffer> output;
    281     const int channels = DetermineChannels(av_frame_.get());
    282     if (frame_decoded) {
    283       if (av_frame_->sample_rate != config_.samples_per_second() ||
    284           channels != ChannelLayoutToChannelCount(config_.channel_layout()) ||
    285           av_frame_->format != av_sample_format_) {
    286         DLOG(ERROR) << "Unsupported midstream configuration change!"
    287                     << " Sample Rate: " << av_frame_->sample_rate << " vs "
    288                     << config_.samples_per_second()
    289                     << ", Channels: " << channels << " vs "
    290                     << ChannelLayoutToChannelCount(config_.channel_layout())
    291                     << ", Sample Format: " << av_frame_->format << " vs "
    292                     << av_sample_format_;
    293 
    294         if (config_.codec() == kCodecAAC &&
    295             av_frame_->sample_rate == 2 * config_.samples_per_second()) {
    296           MEDIA_LOG(log_cb_) << "Implicit HE-AAC signalling is being used."
    297                              << " Please use mp4a.40.5 instead of mp4a.40.2 in"
    298                              << " the mimetype.";
    299         }
    300         // This is an unrecoverable error, so bail out.
    301         av_frame_unref(av_frame_.get());
    302         return false;
    303       }
    304 
    305       // Get the AudioBuffer that the data was decoded into. Adjust the number
    306       // of frames, in case fewer than requested were actually decoded.
    307       output = reinterpret_cast<AudioBuffer*>(
    308           av_buffer_get_opaque(av_frame_->buf[0]));
    309 
    310       DCHECK_EQ(ChannelLayoutToChannelCount(config_.channel_layout()),
    311                 output->channel_count());
    312       const int unread_frames = output->frame_count() - av_frame_->nb_samples;
    313       DCHECK_GE(unread_frames, 0);
    314       if (unread_frames > 0)
    315         output->TrimEnd(unread_frames);
    316       av_frame_unref(av_frame_.get());
    317     }
    318 
    319     // WARNING: |av_frame_| no longer has valid data at this point.
    320     const int decoded_frames = frame_decoded ? output->frame_count() : 0;
    321     if (IsEndOfStream(result, decoded_frames, buffer)) {
    322       DCHECK_EQ(packet.size, 0);
    323     } else if (discard_helper_->ProcessBuffers(buffer, output)) {
    324       *has_produced_frame = true;
    325       output_cb_.Run(output);
    326     }
    327   } while (packet.size > 0);
    328 
    329   return true;
    330 }
    331 
    332 void FFmpegAudioDecoder::ReleaseFFmpegResources() {
    333   codec_context_.reset();
    334   av_frame_.reset();
    335 }
    336 
    337 bool FFmpegAudioDecoder::ConfigureDecoder() {
    338   if (!config_.IsValidConfig()) {
    339     DLOG(ERROR) << "Invalid audio stream -"
    340                 << " codec: " << config_.codec()
    341                 << " channel layout: " << config_.channel_layout()
    342                 << " bits per channel: " << config_.bits_per_channel()
    343                 << " samples per second: " << config_.samples_per_second();
    344     return false;
    345   }
    346 
    347   if (config_.is_encrypted()) {
    348     DLOG(ERROR) << "Encrypted audio stream not supported";
    349     return false;
    350   }
    351 
    352   // Release existing decoder resources if necessary.
    353   ReleaseFFmpegResources();
    354 
    355   // Initialize AVCodecContext structure.
    356   codec_context_.reset(avcodec_alloc_context3(NULL));
    357   AudioDecoderConfigToAVCodecContext(config_, codec_context_.get());
    358 
    359   codec_context_->opaque = this;
    360   codec_context_->get_buffer2 = GetAudioBuffer;
    361   codec_context_->refcounted_frames = 1;
    362 
    363   AVCodec* codec = avcodec_find_decoder(codec_context_->codec_id);
    364   if (!codec || avcodec_open2(codec_context_.get(), codec, NULL) < 0) {
    365     DLOG(ERROR) << "Could not initialize audio decoder: "
    366                 << codec_context_->codec_id;
    367     ReleaseFFmpegResources();
    368     state_ = kUninitialized;
    369     return false;
    370   }
    371 
    372   // Success!
    373   av_frame_.reset(av_frame_alloc());
    374   discard_helper_.reset(new AudioDiscardHelper(config_.samples_per_second(),
    375                                                config_.codec_delay()));
    376   av_sample_format_ = codec_context_->sample_fmt;
    377 
    378   if (codec_context_->channels !=
    379       ChannelLayoutToChannelCount(config_.channel_layout())) {
    380     DLOG(ERROR) << "Audio configuration specified "
    381                 << ChannelLayoutToChannelCount(config_.channel_layout())
    382                 << " channels, but FFmpeg thinks the file contains "
    383                 << codec_context_->channels << " channels";
    384     ReleaseFFmpegResources();
    385     state_ = kUninitialized;
    386     return false;
    387   }
    388 
    389   ResetTimestampState();
    390   return true;
    391 }
    392 
    393 void FFmpegAudioDecoder::ResetTimestampState() {
    394   discard_helper_->Reset(config_.codec_delay());
    395 }
    396 
    397 }  // namespace media
    398