Home | History | Annotate | Download | only in android 
      1 // Copyright 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "media/base/android/media_decoder_job.h"
      6 
      7 #include "base/bind.h"
      8 #include "base/callback_helpers.h"
      9 #include "base/debug/trace_event.h"
     10 #include "base/message_loop/message_loop.h"
     11 #include "media/base/android/media_codec_bridge.h"
     12 #include "media/base/bind_to_loop.h"
     13 #include "media/base/buffers.h"
     14 
     15 namespace media {
     16 
     17 // Timeout value for media codec operations. Because the first
     18 // DequeInputBuffer() can take about 150 milliseconds, use 250 milliseconds
     19 // here. See http://b/9357571.
     20 static const int kMediaCodecTimeoutInMilliseconds = 250;
     21 
     22 MediaDecoderJob::MediaDecoderJob(
     23     const scoped_refptr<base::MessageLoopProxy>& decoder_loop,
     24     MediaCodecBridge* media_codec_bridge,
     25     const base::Closure& request_data_cb)
     26     : ui_loop_(base::MessageLoopProxy::current()),
     27       decoder_loop_(decoder_loop),
     28       media_codec_bridge_(media_codec_bridge),
     29       needs_flush_(false),
     30       input_eos_encountered_(false),
     31       output_eos_encountered_(false),
     32       skip_eos_enqueue_(true),
     33       prerolling_(true),
     34       weak_this_(this),
     35       request_data_cb_(request_data_cb),
     36       access_unit_index_(0),
     37       input_buf_index_(-1),
     38       stop_decode_pending_(false),
     39       destroy_pending_(false) {
     40 }
     41 
     42 MediaDecoderJob::~MediaDecoderJob() {}
     43 
     44 void MediaDecoderJob::OnDataReceived(const DemuxerData& data) {
     45   DVLOG(1) << __FUNCTION__ << ": " << data.access_units.size() << " units";
     46   DCHECK(ui_loop_->BelongsToCurrentThread());
     47   DCHECK(!on_data_received_cb_.is_null());
     48 
     49   TRACE_EVENT_ASYNC_END2(
     50       "media", "MediaDecoderJob::RequestData", this,
     51       "Data type", data.type == media::DemuxerStream::AUDIO ? "AUDIO" : "VIDEO",
     52       "Units read", data.access_units.size());
     53 
     54   base::Closure done_cb = base::ResetAndReturn(&on_data_received_cb_);
     55 
     56   if (stop_decode_pending_) {
     57     OnDecodeCompleted(MEDIA_CODEC_STOPPED, kNoTimestamp(), 0);
     58     return;
     59   }
     60 
     61   access_unit_index_ = 0;
     62   received_data_ = data;
     63   done_cb.Run();
     64 }
     65 
     66 void MediaDecoderJob::Prefetch(const base::Closure& prefetch_cb) {
     67   DCHECK(ui_loop_->BelongsToCurrentThread());
     68   DCHECK(on_data_received_cb_.is_null());
     69   DCHECK(decode_cb_.is_null());
     70 
     71   if (HasData()) {
     72     DVLOG(1) << __FUNCTION__ << " : using previously received data";
     73     ui_loop_->PostTask(FROM_HERE, prefetch_cb);
     74     return;
     75   }
     76 
     77   DVLOG(1) << __FUNCTION__ << " : requesting data";
     78   RequestData(prefetch_cb);
     79 }
     80 
     81 bool MediaDecoderJob::Decode(
     82     const base::TimeTicks& start_time_ticks,
     83     const base::TimeDelta& start_presentation_timestamp,
     84     const DecoderCallback& callback) {
     85   DCHECK(decode_cb_.is_null());
     86   DCHECK(on_data_received_cb_.is_null());
     87   DCHECK(ui_loop_->BelongsToCurrentThread());
     88 
     89   decode_cb_ = callback;
     90 
     91   if (!HasData()) {
     92     RequestData(base::Bind(&MediaDecoderJob::DecodeNextAccessUnit,
     93                            base::Unretained(this),
     94                            start_time_ticks,
     95                            start_presentation_timestamp));
     96     return true;
     97   }
     98 
     99   if (DemuxerStream::kConfigChanged ==
    100       received_data_.access_units[access_unit_index_].status) {
    101     // Clear received data because we need to handle a config change.
    102     decode_cb_.Reset();
    103     received_data_ = DemuxerData();
    104     access_unit_index_ = 0;
    105     return false;
    106   }
    107 
    108   DecodeNextAccessUnit(start_time_ticks, start_presentation_timestamp);
    109   return true;
    110 }
    111 
    112 void MediaDecoderJob::StopDecode() {
    113   DCHECK(ui_loop_->BelongsToCurrentThread());
    114   DCHECK(is_decoding());
    115   stop_decode_pending_ = true;
    116 }
    117 
    118 void MediaDecoderJob::Flush() {
    119   DCHECK(decode_cb_.is_null());
    120 
    121   // Do nothing, flush when the next Decode() happens.
    122   needs_flush_ = true;
    123   received_data_ = DemuxerData();
    124   input_eos_encountered_ = false;
    125   access_unit_index_ = 0;
    126   on_data_received_cb_.Reset();
    127 }
    128 
    129 void MediaDecoderJob::BeginPrerolling(
    130     const base::TimeDelta& preroll_timestamp) {
    131   DVLOG(1) << __FUNCTION__ << "(" << preroll_timestamp.InSecondsF() << ")";
    132   DCHECK(ui_loop_->BelongsToCurrentThread());
    133   DCHECK(!is_decoding());
    134 
    135   preroll_timestamp_ = preroll_timestamp;
    136   prerolling_ = true;
    137 }
    138 
    139 void MediaDecoderJob::Release() {
    140   DCHECK(ui_loop_->BelongsToCurrentThread());
    141   DVLOG(1) << __FUNCTION__;
    142 
    143   // If the decoder job is not waiting for data, and is still decoding, we
    144   // cannot delete the job immediately.
    145   destroy_pending_ = on_data_received_cb_.is_null() && is_decoding();
    146 
    147   request_data_cb_.Reset();
    148   on_data_received_cb_.Reset();
    149   decode_cb_.Reset();
    150 
    151   if (destroy_pending_) {
    152     DVLOG(1) << __FUNCTION__ << " : delete is pending decode completion";
    153     return;
    154   }
    155 
    156   delete this;
    157 }
    158 
    159 MediaCodecStatus MediaDecoderJob::QueueInputBuffer(const AccessUnit& unit) {
    160   DVLOG(1) << __FUNCTION__;
    161   DCHECK(decoder_loop_->BelongsToCurrentThread());
    162   TRACE_EVENT0("media", __FUNCTION__);
    163 
    164   int input_buf_index = input_buf_index_;
    165   input_buf_index_ = -1;
    166 
    167   // TODO(xhwang): Hide DequeueInputBuffer() and the index in MediaCodecBridge.
    168   if (input_buf_index == -1) {
    169     base::TimeDelta timeout = base::TimeDelta::FromMilliseconds(
    170         kMediaCodecTimeoutInMilliseconds);
    171     MediaCodecStatus status =
    172         media_codec_bridge_->DequeueInputBuffer(timeout, &input_buf_index);
    173     if (status != MEDIA_CODEC_OK) {
    174       DVLOG(1) << "DequeueInputBuffer fails: " << status;
    175       return status;
    176     }
    177   }
    178 
    179   // TODO(qinmin): skip frames if video is falling far behind.
    180   DCHECK_GE(input_buf_index, 0);
    181   if (unit.end_of_stream || unit.data.empty()) {
    182     media_codec_bridge_->QueueEOS(input_buf_index);
    183     return MEDIA_CODEC_INPUT_END_OF_STREAM;
    184   }
    185 
    186   if (unit.key_id.empty() || unit.iv.empty()) {
    187     DCHECK(unit.iv.empty() || !unit.key_id.empty());
    188     return media_codec_bridge_->QueueInputBuffer(
    189         input_buf_index, &unit.data[0], unit.data.size(), unit.timestamp);
    190   }
    191 
    192   MediaCodecStatus status = media_codec_bridge_->QueueSecureInputBuffer(
    193       input_buf_index,
    194       &unit.data[0], unit.data.size(),
    195       reinterpret_cast<const uint8*>(&unit.key_id[0]), unit.key_id.size(),
    196       reinterpret_cast<const uint8*>(&unit.iv[0]), unit.iv.size(),
    197       unit.subsamples.empty() ? NULL : &unit.subsamples[0],
    198       unit.subsamples.size(),
    199       unit.timestamp);
    200 
    201   // In case of MEDIA_CODEC_NO_KEY, we must reuse the |input_buf_index_|.
    202   // Otherwise MediaDrm will report errors.
    203   if (status == MEDIA_CODEC_NO_KEY)
    204     input_buf_index_ = input_buf_index;
    205 
    206   return status;
    207 }
    208 
    209 bool MediaDecoderJob::HasData() const {
    210   DCHECK(ui_loop_->BelongsToCurrentThread());
    211   // When |input_eos_encountered_| is set, |access_units| must not be empty and
    212   // |access_unit_index_| must be pointing to an EOS unit. We'll reuse this
    213   // unit to flush the decoder until we hit output EOS.
    214   DCHECK(!input_eos_encountered_ ||
    215          (received_data_.access_units.size() > 0 &&
    216           access_unit_index_ < received_data_.access_units.size()))
    217       << " (access_units.size(): " << received_data_.access_units.size()
    218       << ", access_unit_index_: " << access_unit_index_ << ")";
    219   return access_unit_index_ < received_data_.access_units.size() ||
    220       input_eos_encountered_;
    221 }
    222 
    223 void MediaDecoderJob::RequestData(const base::Closure& done_cb) {
    224   DVLOG(1) << __FUNCTION__;
    225   DCHECK(ui_loop_->BelongsToCurrentThread());
    226   DCHECK(on_data_received_cb_.is_null());
    227   DCHECK(!input_eos_encountered_);
    228 
    229   TRACE_EVENT_ASYNC_BEGIN0("media", "MediaDecoderJob::RequestData", this);
    230 
    231   received_data_ = DemuxerData();
    232   access_unit_index_ = 0;
    233   on_data_received_cb_ = done_cb;
    234 
    235   request_data_cb_.Run();
    236 }
    237 
    238 void MediaDecoderJob::DecodeNextAccessUnit(
    239     const base::TimeTicks& start_time_ticks,
    240     const base::TimeDelta& start_presentation_timestamp) {
    241   DCHECK(ui_loop_->BelongsToCurrentThread());
    242   DCHECK(!decode_cb_.is_null());
    243 
    244   // If the first access unit is a config change, request the player to dequeue
    245   // the input buffer again so that it can request config data.
    246   if (received_data_.access_units[access_unit_index_].status ==
    247       DemuxerStream::kConfigChanged) {
    248     ui_loop_->PostTask(FROM_HERE,
    249                        base::Bind(&MediaDecoderJob::OnDecodeCompleted,
    250                                   base::Unretained(this),
    251                                   MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER,
    252                                   kNoTimestamp(),
    253                                   0));
    254     return;
    255   }
    256 
    257   decoder_loop_->PostTask(FROM_HERE, base::Bind(
    258       &MediaDecoderJob::DecodeInternal, base::Unretained(this),
    259       received_data_.access_units[access_unit_index_],
    260       start_time_ticks, start_presentation_timestamp, needs_flush_,
    261       media::BindToLoop(ui_loop_, base::Bind(
    262           &MediaDecoderJob::OnDecodeCompleted, base::Unretained(this)))));
    263   needs_flush_ = false;
    264 }
    265 
    266 void MediaDecoderJob::DecodeInternal(
    267     const AccessUnit& unit,
    268     const base::TimeTicks& start_time_ticks,
    269     const base::TimeDelta& start_presentation_timestamp,
    270     bool needs_flush,
    271     const MediaDecoderJob::DecoderCallback& callback) {
    272   DVLOG(1) << __FUNCTION__;
    273   DCHECK(decoder_loop_->BelongsToCurrentThread());
    274   TRACE_EVENT0("media", __FUNCTION__);
    275 
    276   if (needs_flush) {
    277     DVLOG(1) << "DecodeInternal needs flush.";
    278     input_eos_encountered_ = false;
    279     output_eos_encountered_ = false;
    280     MediaCodecStatus reset_status = media_codec_bridge_->Reset();
    281     if (MEDIA_CODEC_OK != reset_status) {
    282       callback.Run(reset_status, kNoTimestamp(), 0);
    283       return;
    284     }
    285   }
    286 
    287   // Once output EOS has occurred, we should not be asked to decode again.
    288   // MediaCodec has undefined behavior if similarly asked to decode after output
    289   // EOS.
    290   DCHECK(!output_eos_encountered_);
    291 
    292   // For aborted access unit, just skip it and inform the player.
    293   if (unit.status == DemuxerStream::kAborted) {
    294     // TODO(qinmin): use a new enum instead of MEDIA_CODEC_STOPPED.
    295     callback.Run(MEDIA_CODEC_STOPPED, kNoTimestamp(), 0);
    296     return;
    297   }
    298 
    299   if (skip_eos_enqueue_) {
    300     if (unit.end_of_stream || unit.data.empty()) {
    301       input_eos_encountered_ = true;
    302       output_eos_encountered_ = true;
    303       callback.Run(MEDIA_CODEC_OUTPUT_END_OF_STREAM, kNoTimestamp(), 0);
    304       return;
    305     }
    306 
    307     skip_eos_enqueue_ = false;
    308   }
    309 
    310   MediaCodecStatus input_status = MEDIA_CODEC_INPUT_END_OF_STREAM;
    311   if (!input_eos_encountered_) {
    312     input_status = QueueInputBuffer(unit);
    313     if (input_status == MEDIA_CODEC_INPUT_END_OF_STREAM) {
    314       input_eos_encountered_ = true;
    315     } else if (input_status != MEDIA_CODEC_OK) {
    316       callback.Run(input_status, kNoTimestamp(), 0);
    317       return;
    318     }
    319   }
    320 
    321   int buffer_index = 0;
    322   size_t offset = 0;
    323   size_t size = 0;
    324   base::TimeDelta presentation_timestamp;
    325 
    326   base::TimeDelta timeout = base::TimeDelta::FromMilliseconds(
    327       kMediaCodecTimeoutInMilliseconds);
    328 
    329   MediaCodecStatus status =
    330       media_codec_bridge_->DequeueOutputBuffer(timeout,
    331                                                &buffer_index,
    332                                                &offset,
    333                                                &size,
    334                                                &presentation_timestamp,
    335                                                &output_eos_encountered_,
    336                                                NULL);
    337 
    338   if (status != MEDIA_CODEC_OK) {
    339     if (status == MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED &&
    340         !media_codec_bridge_->GetOutputBuffers()) {
    341       status = MEDIA_CODEC_ERROR;
    342     }
    343     callback.Run(status, kNoTimestamp(), 0);
    344     return;
    345   }
    346 
    347   // TODO(xhwang/qinmin): This logic is correct but strange. Clean it up.
    348   if (output_eos_encountered_)
    349     status = MEDIA_CODEC_OUTPUT_END_OF_STREAM;
    350   else if (input_status == MEDIA_CODEC_INPUT_END_OF_STREAM)
    351     status = MEDIA_CODEC_INPUT_END_OF_STREAM;
    352 
    353   // Check whether we need to render the output.
    354   // TODO(qinmin): comparing most recently queued input's |unit.timestamp| with
    355   // |preroll_timestamp_| is not accurate due to data reordering and possible
    356   // input queueing without immediate dequeue when |input_status| !=
    357   // |MEDIA_CODEC_OK|. Need to use the |presentation_timestamp| for video, and
    358   // use |size| to calculate the timestamp for audio. See
    359   // http://crbug.com/310823 and http://b/11356652.
    360   bool render_output  = unit.timestamp >= preroll_timestamp_ &&
    361       (status != MEDIA_CODEC_OUTPUT_END_OF_STREAM || size != 0u);
    362   base::TimeDelta time_to_render;
    363   DCHECK(!start_time_ticks.is_null());
    364   if (render_output && ComputeTimeToRender()) {
    365     time_to_render = presentation_timestamp - (base::TimeTicks::Now() -
    366         start_time_ticks + start_presentation_timestamp);
    367   }
    368 
    369   if (time_to_render > base::TimeDelta()) {
    370     decoder_loop_->PostDelayedTask(
    371         FROM_HERE,
    372         base::Bind(&MediaDecoderJob::ReleaseOutputBuffer,
    373                    weak_this_.GetWeakPtr(), buffer_index, size, render_output,
    374                    base::Bind(callback, status, presentation_timestamp)),
    375         time_to_render);
    376     return;
    377   }
    378 
    379   // TODO(qinmin): The codec is lagging behind, need to recalculate the
    380   // |start_presentation_timestamp_| and |start_time_ticks_| in
    381   // media_source_player.cc.
    382   DVLOG(1) << "codec is lagging behind :" << time_to_render.InMicroseconds();
    383   if (render_output) {
    384     // The player won't expect a timestamp smaller than the
    385     // |start_presentation_timestamp|. However, this could happen due to decoder
    386     // errors.
    387     presentation_timestamp = std::max(
    388         presentation_timestamp, start_presentation_timestamp);
    389   } else {
    390     presentation_timestamp = kNoTimestamp();
    391   }
    392   ReleaseOutputCompletionCallback completion_callback = base::Bind(
    393       callback, status, presentation_timestamp);
    394   ReleaseOutputBuffer(buffer_index, size, render_output, completion_callback);
    395 }
    396 
    397 void MediaDecoderJob::OnDecodeCompleted(
    398     MediaCodecStatus status, const base::TimeDelta& presentation_timestamp,
    399     size_t audio_output_bytes) {
    400   DCHECK(ui_loop_->BelongsToCurrentThread());
    401 
    402   if (destroy_pending_) {
    403     DVLOG(1) << __FUNCTION__ << " : completing pending deletion";
    404     delete this;
    405     return;
    406   }
    407 
    408   DCHECK(!decode_cb_.is_null());
    409 
    410   // If output was queued for rendering, then we have completed prerolling.
    411   if (presentation_timestamp != kNoTimestamp())
    412     prerolling_ = false;
    413 
    414   switch (status) {
    415     case MEDIA_CODEC_OK:
    416     case MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER:
    417     case MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED:
    418     case MEDIA_CODEC_OUTPUT_FORMAT_CHANGED:
    419     case MEDIA_CODEC_OUTPUT_END_OF_STREAM:
    420       if (!input_eos_encountered_)
    421         access_unit_index_++;
    422       break;
    423 
    424     case MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER:
    425     case MEDIA_CODEC_INPUT_END_OF_STREAM:
    426     case MEDIA_CODEC_NO_KEY:
    427     case MEDIA_CODEC_STOPPED:
    428     case MEDIA_CODEC_ERROR:
    429       // Do nothing.
    430       break;
    431   };
    432 
    433   stop_decode_pending_ = false;
    434   base::ResetAndReturn(&decode_cb_).Run(status, presentation_timestamp,
    435                                         audio_output_bytes);
    436 }
    437 
    438 }  // namespace media
    439