Home | History | Annotate | Download | only in mp4 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "media/mp4/mp4_stream_parser.h"
      6 
      7 #include "base/callback.h"
      8 #include "base/callback_helpers.h"
      9 #include "base/logging.h"
     10 #include "base/time/time.h"
     11 #include "media/base/audio_decoder_config.h"
     12 #include "media/base/stream_parser_buffer.h"
     13 #include "media/base/video_decoder_config.h"
     14 #include "media/base/video_util.h"
     15 #include "media/mp4/box_definitions.h"
     16 #include "media/mp4/box_reader.h"
     17 #include "media/mp4/es_descriptor.h"
     18 #include "media/mp4/rcheck.h"
     19 
     20 namespace media {
     21 namespace mp4 {
     22 
     23 // TODO(xhwang): Figure out the init data type appropriately once it's spec'ed.
     24 static const char kMp4InitDataType[] = "video/mp4";
     25 
     26 MP4StreamParser::MP4StreamParser(const std::set<int>& audio_object_types,
     27                                  bool has_sbr)
     28     : state_(kWaitingForInit),
     29       moof_head_(0),
     30       mdat_tail_(0),
     31       has_audio_(false),
     32       has_video_(false),
     33       audio_track_id_(0),
     34       video_track_id_(0),
     35       audio_object_types_(audio_object_types),
     36       has_sbr_(has_sbr),
     37       is_audio_track_encrypted_(false),
     38       is_video_track_encrypted_(false) {
     39 }
     40 
     41 MP4StreamParser::~MP4StreamParser() {}
     42 
     43 void MP4StreamParser::Init(const InitCB& init_cb,
     44                            const NewConfigCB& config_cb,
     45                            const NewBuffersCB& new_buffers_cb,
     46                            const NewTextBuffersCB& /* text_cb */ ,
     47                            const NeedKeyCB& need_key_cb,
     48                            const AddTextTrackCB& /* add_text_track_cb */ ,
     49                            const NewMediaSegmentCB& new_segment_cb,
     50                            const base::Closure& end_of_segment_cb,
     51                            const LogCB& log_cb) {
     52   DCHECK_EQ(state_, kWaitingForInit);
     53   DCHECK(init_cb_.is_null());
     54   DCHECK(!init_cb.is_null());
     55   DCHECK(!config_cb.is_null());
     56   DCHECK(!new_buffers_cb.is_null());
     57   DCHECK(!need_key_cb.is_null());
     58   DCHECK(!end_of_segment_cb.is_null());
     59 
     60   ChangeState(kParsingBoxes);
     61   init_cb_ = init_cb;
     62   config_cb_ = config_cb;
     63   new_buffers_cb_ = new_buffers_cb;
     64   need_key_cb_ = need_key_cb;
     65   new_segment_cb_ = new_segment_cb;
     66   end_of_segment_cb_ = end_of_segment_cb;
     67   log_cb_ = log_cb;
     68 }
     69 
     70 void MP4StreamParser::Reset() {
     71   queue_.Reset();
     72   runs_.reset();
     73   moof_head_ = 0;
     74   mdat_tail_ = 0;
     75 }
     76 
     77 void MP4StreamParser::Flush() {
     78   DCHECK_NE(state_, kWaitingForInit);
     79   Reset();
     80   ChangeState(kParsingBoxes);
     81 }
     82 
     83 bool MP4StreamParser::Parse(const uint8* buf, int size) {
     84   DCHECK_NE(state_, kWaitingForInit);
     85 
     86   if (state_ == kError)
     87     return false;
     88 
     89   queue_.Push(buf, size);
     90 
     91   BufferQueue audio_buffers;
     92   BufferQueue video_buffers;
     93 
     94   bool result, err = false;
     95 
     96   do {
     97     if (state_ == kParsingBoxes) {
     98       result = ParseBox(&err);
     99     } else {
    100       DCHECK_EQ(kEmittingSamples, state_);
    101       result = EnqueueSample(&audio_buffers, &video_buffers, &err);
    102       if (result) {
    103         int64 max_clear = runs_->GetMaxClearOffset() + moof_head_;
    104         err = !ReadAndDiscardMDATsUntil(max_clear);
    105       }
    106     }
    107   } while (result && !err);
    108 
    109   if (!err)
    110     err = !SendAndFlushSamples(&audio_buffers, &video_buffers);
    111 
    112   if (err) {
    113     DLOG(ERROR) << "Error while parsing MP4";
    114     moov_.reset();
    115     Reset();
    116     ChangeState(kError);
    117     return false;
    118   }
    119 
    120   return true;
    121 }
    122 
    123 bool MP4StreamParser::ParseBox(bool* err) {
    124   const uint8* buf;
    125   int size;
    126   queue_.Peek(&buf, &size);
    127   if (!size) return false;
    128 
    129   scoped_ptr<BoxReader> reader(
    130       BoxReader::ReadTopLevelBox(buf, size, log_cb_, err));
    131   if (reader.get() == NULL) return false;
    132 
    133   if (reader->type() == FOURCC_MOOV) {
    134     *err = !ParseMoov(reader.get());
    135   } else if (reader->type() == FOURCC_MOOF) {
    136     moof_head_ = queue_.head();
    137     *err = !ParseMoof(reader.get());
    138 
    139     // Set up first mdat offset for ReadMDATsUntil().
    140     mdat_tail_ = queue_.head() + reader->size();
    141 
    142     // Return early to avoid evicting 'moof' data from queue. Auxiliary info may
    143     // be located anywhere in the file, including inside the 'moof' itself.
    144     // (Since 'default-base-is-moof' is mandated, no data references can come
    145     // before the head of the 'moof', so keeping this box around is sufficient.)
    146     return !(*err);
    147   } else {
    148     MEDIA_LOG(log_cb_) << "Skipping unrecognized top-level box: "
    149                        << FourCCToString(reader->type());
    150   }
    151 
    152   queue_.Pop(reader->size());
    153   return !(*err);
    154 }
    155 
    156 
    157 bool MP4StreamParser::ParseMoov(BoxReader* reader) {
    158   moov_.reset(new Movie);
    159   RCHECK(moov_->Parse(reader));
    160   runs_.reset();
    161 
    162   has_audio_ = false;
    163   has_video_ = false;
    164 
    165   AudioDecoderConfig audio_config;
    166   VideoDecoderConfig video_config;
    167 
    168   for (std::vector<Track>::const_iterator track = moov_->tracks.begin();
    169        track != moov_->tracks.end(); ++track) {
    170     // TODO(strobe): Only the first audio and video track present in a file are
    171     // used. (Track selection is better accomplished via Source IDs, though, so
    172     // adding support for track selection within a stream is low-priority.)
    173     const SampleDescription& samp_descr =
    174         track->media.information.sample_table.description;
    175 
    176     // TODO(strobe): When codec reconfigurations are supported, detect and send
    177     // a codec reconfiguration for fragments using a sample description index
    178     // different from the previous one
    179     size_t desc_idx = 0;
    180     for (size_t t = 0; t < moov_->extends.tracks.size(); t++) {
    181       const TrackExtends& trex = moov_->extends.tracks[t];
    182       if (trex.track_id == track->header.track_id) {
    183         desc_idx = trex.default_sample_description_index;
    184         break;
    185       }
    186     }
    187     RCHECK(desc_idx > 0);
    188     desc_idx -= 1;  // BMFF descriptor index is one-based
    189 
    190     if (track->media.handler.type == kAudio && !audio_config.IsValidConfig()) {
    191       RCHECK(!samp_descr.audio_entries.empty());
    192 
    193       // It is not uncommon to find otherwise-valid files with incorrect sample
    194       // description indices, so we fail gracefully in that case.
    195       if (desc_idx >= samp_descr.audio_entries.size())
    196         desc_idx = 0;
    197       const AudioSampleEntry& entry = samp_descr.audio_entries[desc_idx];
    198       const AAC& aac = entry.esds.aac;
    199 
    200       if (!(entry.format == FOURCC_MP4A || entry.format == FOURCC_EAC3 ||
    201             (entry.format == FOURCC_ENCA &&
    202              entry.sinf.format.format == FOURCC_MP4A))) {
    203         MEDIA_LOG(log_cb_) << "Unsupported audio format 0x"
    204                            << std::hex << entry.format << " in stsd box.";
    205         return false;
    206       }
    207 
    208       uint8 audio_type = entry.esds.object_type;
    209       DVLOG(1) << "audio_type " << std::hex << audio_type;
    210       if (audio_type == kForbidden && entry.format == FOURCC_EAC3) {
    211         audio_type = kEAC3;
    212       }
    213       if (audio_object_types_.find(audio_type) == audio_object_types_.end()) {
    214         MEDIA_LOG(log_cb_) << "audio object type 0x" << std::hex << audio_type
    215                            << " does not match what is specified in the"
    216                            << " mimetype.";
    217         return false;
    218       }
    219 
    220       AudioCodec codec = kUnknownAudioCodec;
    221       ChannelLayout channel_layout = CHANNEL_LAYOUT_NONE;
    222       int sample_per_second = 0;
    223       std::vector<uint8> extra_data;
    224       // Check if it is MPEG4 AAC defined in ISO 14496 Part 3 or
    225       // supported MPEG2 AAC varients.
    226       if (ESDescriptor::IsAAC(audio_type)) {
    227         codec = kCodecAAC;
    228         channel_layout = aac.GetChannelLayout(has_sbr_);
    229         sample_per_second = aac.GetOutputSamplesPerSecond(has_sbr_);
    230 #if defined(OS_ANDROID)
    231         extra_data = aac.codec_specific_data();
    232 #endif
    233       } else if (audio_type == kEAC3) {
    234         codec = kCodecEAC3;
    235         channel_layout = GuessChannelLayout(entry.channelcount);
    236         sample_per_second = entry.samplerate;
    237       } else {
    238         MEDIA_LOG(log_cb_) << "Unsupported audio object type 0x" << std::hex
    239                            << audio_type << " in esds.";
    240         return false;
    241       }
    242 
    243       SampleFormat sample_format;
    244       if (entry.samplesize == 8) {
    245         sample_format = kSampleFormatU8;
    246       } else if (entry.samplesize == 16) {
    247         sample_format = kSampleFormatS16;
    248       } else if (entry.samplesize == 32) {
    249         sample_format = kSampleFormatS32;
    250       } else {
    251         LOG(ERROR) << "Unsupported sample size.";
    252         return false;
    253       }
    254 
    255       is_audio_track_encrypted_ = entry.sinf.info.track_encryption.is_encrypted;
    256       DVLOG(1) << "is_audio_track_encrypted_: " << is_audio_track_encrypted_;
    257       audio_config.Initialize(
    258           codec, sample_format, channel_layout, sample_per_second,
    259           extra_data.size() ? &extra_data[0] : NULL, extra_data.size(),
    260           is_audio_track_encrypted_, false);
    261       has_audio_ = true;
    262       audio_track_id_ = track->header.track_id;
    263     }
    264     if (track->media.handler.type == kVideo && !video_config.IsValidConfig()) {
    265       RCHECK(!samp_descr.video_entries.empty());
    266       if (desc_idx >= samp_descr.video_entries.size())
    267         desc_idx = 0;
    268       const VideoSampleEntry& entry = samp_descr.video_entries[desc_idx];
    269 
    270       if (!(entry.format == FOURCC_AVC1 ||
    271             (entry.format == FOURCC_ENCV &&
    272              entry.sinf.format.format == FOURCC_AVC1))) {
    273         MEDIA_LOG(log_cb_) << "Unsupported video format 0x"
    274                            << std::hex << entry.format << " in stsd box.";
    275         return false;
    276       }
    277 
    278       // TODO(strobe): Recover correct crop box
    279       gfx::Size coded_size(entry.width, entry.height);
    280       gfx::Rect visible_rect(coded_size);
    281       gfx::Size natural_size = GetNaturalSize(visible_rect.size(),
    282                                               entry.pixel_aspect.h_spacing,
    283                                               entry.pixel_aspect.v_spacing);
    284       is_video_track_encrypted_ = entry.sinf.info.track_encryption.is_encrypted;
    285       DVLOG(1) << "is_video_track_encrypted_: " << is_video_track_encrypted_;
    286       video_config.Initialize(kCodecH264, H264PROFILE_MAIN,  VideoFrame::YV12,
    287                               coded_size, visible_rect, natural_size,
    288                               // No decoder-specific buffer needed for AVC;
    289                               // SPS/PPS are embedded in the video stream
    290                               NULL, 0, is_video_track_encrypted_, true);
    291       has_video_ = true;
    292       video_track_id_ = track->header.track_id;
    293     }
    294   }
    295 
    296   RCHECK(config_cb_.Run(audio_config, video_config));
    297 
    298   base::TimeDelta duration;
    299   if (moov_->extends.header.fragment_duration > 0) {
    300     duration = TimeDeltaFromRational(moov_->extends.header.fragment_duration,
    301                                      moov_->header.timescale);
    302   } else if (moov_->header.duration > 0 &&
    303              moov_->header.duration != kuint64max) {
    304     duration = TimeDeltaFromRational(moov_->header.duration,
    305                                      moov_->header.timescale);
    306   } else {
    307     duration = kInfiniteDuration();
    308   }
    309 
    310   if (!init_cb_.is_null())
    311     base::ResetAndReturn(&init_cb_).Run(true, duration);
    312 
    313   EmitNeedKeyIfNecessary(moov_->pssh);
    314   return true;
    315 }
    316 
    317 bool MP4StreamParser::ParseMoof(BoxReader* reader) {
    318   RCHECK(moov_.get());  // Must already have initialization segment
    319   MovieFragment moof;
    320   RCHECK(moof.Parse(reader));
    321   if (!runs_)
    322     runs_.reset(new TrackRunIterator(moov_.get(), log_cb_));
    323   RCHECK(runs_->Init(moof));
    324   EmitNeedKeyIfNecessary(moof.pssh);
    325   new_segment_cb_.Run();
    326   ChangeState(kEmittingSamples);
    327   return true;
    328 }
    329 
    330 void MP4StreamParser::EmitNeedKeyIfNecessary(
    331     const std::vector<ProtectionSystemSpecificHeader>& headers) {
    332   // TODO(strobe): ensure that the value of init_data (all PSSH headers
    333   // concatenated in arbitrary order) matches the EME spec.
    334   // See https://www.w3.org/Bugs/Public/show_bug.cgi?id=17673.
    335   if (headers.empty())
    336     return;
    337 
    338   size_t total_size = 0;
    339   for (size_t i = 0; i < headers.size(); i++)
    340     total_size += headers[i].raw_box.size();
    341 
    342   scoped_ptr<uint8[]> init_data(new uint8[total_size]);
    343   size_t pos = 0;
    344   for (size_t i = 0; i < headers.size(); i++) {
    345     memcpy(&init_data.get()[pos], &headers[i].raw_box[0],
    346            headers[i].raw_box.size());
    347     pos += headers[i].raw_box.size();
    348   }
    349   need_key_cb_.Run(kMp4InitDataType, init_data.Pass(), total_size);
    350 }
    351 
    352 bool MP4StreamParser::PrepareAVCBuffer(
    353     const AVCDecoderConfigurationRecord& avc_config,
    354     std::vector<uint8>* frame_buf,
    355     std::vector<SubsampleEntry>* subsamples) const {
    356   // Convert the AVC NALU length fields to Annex B headers, as expected by
    357   // decoding libraries. Since this may enlarge the size of the buffer, we also
    358   // update the clear byte count for each subsample if encryption is used to
    359   // account for the difference in size between the length prefix and Annex B
    360   // start code.
    361   RCHECK(AVC::ConvertFrameToAnnexB(avc_config.length_size, frame_buf));
    362   if (!subsamples->empty()) {
    363     const int nalu_size_diff = 4 - avc_config.length_size;
    364     size_t expected_size = runs_->sample_size() +
    365         subsamples->size() * nalu_size_diff;
    366     RCHECK(frame_buf->size() == expected_size);
    367     for (size_t i = 0; i < subsamples->size(); i++)
    368       (*subsamples)[i].clear_bytes += nalu_size_diff;
    369   }
    370 
    371   if (runs_->is_keyframe()) {
    372     // If this is a keyframe, we (re-)inject SPS and PPS headers at the start of
    373     // a frame. If subsample info is present, we also update the clear byte
    374     // count for that first subsample.
    375     std::vector<uint8> param_sets;
    376     RCHECK(AVC::ConvertConfigToAnnexB(avc_config, &param_sets));
    377     frame_buf->insert(frame_buf->begin(),
    378                       param_sets.begin(), param_sets.end());
    379     if (!subsamples->empty())
    380       (*subsamples)[0].clear_bytes += param_sets.size();
    381   }
    382   return true;
    383 }
    384 
    385 bool MP4StreamParser::PrepareAACBuffer(
    386     const AAC& aac_config, std::vector<uint8>* frame_buf,
    387     std::vector<SubsampleEntry>* subsamples) const {
    388   // Append an ADTS header to every audio sample.
    389   RCHECK(aac_config.ConvertEsdsToADTS(frame_buf));
    390 
    391   // As above, adjust subsample information to account for the headers. AAC is
    392   // not required to use subsample encryption, so we may need to add an entry.
    393   if (subsamples->empty()) {
    394     SubsampleEntry entry;
    395     entry.clear_bytes = AAC::kADTSHeaderSize;
    396     entry.cypher_bytes = frame_buf->size() - AAC::kADTSHeaderSize;
    397     subsamples->push_back(entry);
    398   } else {
    399     (*subsamples)[0].clear_bytes += AAC::kADTSHeaderSize;
    400   }
    401   return true;
    402 }
    403 
    404 bool MP4StreamParser::EnqueueSample(BufferQueue* audio_buffers,
    405                                     BufferQueue* video_buffers,
    406                                     bool* err) {
    407   if (!runs_->IsRunValid()) {
    408     // Flush any buffers we've gotten in this chunk so that buffers don't
    409     // cross NewSegment() calls
    410     *err = !SendAndFlushSamples(audio_buffers, video_buffers);
    411     if (*err)
    412       return false;
    413 
    414     // Remain in kEnqueueingSamples state, discarding data, until the end of
    415     // the current 'mdat' box has been appended to the queue.
    416     if (!queue_.Trim(mdat_tail_))
    417       return false;
    418 
    419     ChangeState(kParsingBoxes);
    420     end_of_segment_cb_.Run();
    421     return true;
    422   }
    423 
    424   if (!runs_->IsSampleValid()) {
    425     runs_->AdvanceRun();
    426     return true;
    427   }
    428 
    429   DCHECK(!(*err));
    430 
    431   const uint8* buf;
    432   int buf_size;
    433   queue_.Peek(&buf, &buf_size);
    434   if (!buf_size) return false;
    435 
    436   bool audio = has_audio_ && audio_track_id_ == runs_->track_id();
    437   bool video = has_video_ && video_track_id_ == runs_->track_id();
    438 
    439   // Skip this entire track if it's not one we're interested in
    440   if (!audio && !video)
    441     runs_->AdvanceRun();
    442 
    443   // Attempt to cache the auxiliary information first. Aux info is usually
    444   // placed in a contiguous block before the sample data, rather than being
    445   // interleaved. If we didn't cache it, this would require that we retain the
    446   // start of the segment buffer while reading samples. Aux info is typically
    447   // quite small compared to sample data, so this pattern is useful on
    448   // memory-constrained devices where the source buffer consumes a substantial
    449   // portion of the total system memory.
    450   if (runs_->AuxInfoNeedsToBeCached()) {
    451     queue_.PeekAt(runs_->aux_info_offset() + moof_head_, &buf, &buf_size);
    452     if (buf_size < runs_->aux_info_size()) return false;
    453     *err = !runs_->CacheAuxInfo(buf, buf_size);
    454     return !*err;
    455   }
    456 
    457   queue_.PeekAt(runs_->sample_offset() + moof_head_, &buf, &buf_size);
    458   if (buf_size < runs_->sample_size()) return false;
    459 
    460   scoped_ptr<DecryptConfig> decrypt_config;
    461   std::vector<SubsampleEntry> subsamples;
    462   if (runs_->is_encrypted()) {
    463     decrypt_config = runs_->GetDecryptConfig();
    464     if (!decrypt_config) {
    465       *err = true;
    466       return false;
    467     }
    468     subsamples = decrypt_config->subsamples();
    469   }
    470 
    471   std::vector<uint8> frame_buf(buf, buf + runs_->sample_size());
    472   if (video) {
    473     if (!PrepareAVCBuffer(runs_->video_description().avcc,
    474                           &frame_buf, &subsamples)) {
    475       MEDIA_LOG(log_cb_) << "Failed to prepare AVC sample for decode";
    476       *err = true;
    477       return false;
    478     }
    479   }
    480 
    481   if (audio) {
    482     if (ESDescriptor::IsAAC(runs_->audio_description().esds.object_type) &&
    483         !PrepareAACBuffer(runs_->audio_description().esds.aac,
    484                           &frame_buf, &subsamples)) {
    485       MEDIA_LOG(log_cb_) << "Failed to prepare AAC sample for decode";
    486       *err = true;
    487       return false;
    488     }
    489   }
    490 
    491   if (decrypt_config) {
    492     if (!subsamples.empty()) {
    493     // Create a new config with the updated subsamples.
    494     decrypt_config.reset(new DecryptConfig(
    495         decrypt_config->key_id(),
    496         decrypt_config->iv(),
    497         decrypt_config->data_offset(),
    498         subsamples));
    499     }
    500     // else, use the existing config.
    501   } else if ((audio && is_audio_track_encrypted_) ||
    502              (video && is_video_track_encrypted_)) {
    503     // The media pipeline requires a DecryptConfig with an empty |iv|.
    504     // TODO(ddorwin): Refactor so we do not need a fake key ID ("1");
    505     decrypt_config.reset(
    506         new DecryptConfig("1", "", 0, std::vector<SubsampleEntry>()));
    507   }
    508 
    509   scoped_refptr<StreamParserBuffer> stream_buf =
    510     StreamParserBuffer::CopyFrom(&frame_buf[0], frame_buf.size(),
    511                                  runs_->is_keyframe());
    512 
    513   if (decrypt_config)
    514     stream_buf->set_decrypt_config(decrypt_config.Pass());
    515 
    516   stream_buf->set_duration(runs_->duration());
    517   stream_buf->set_timestamp(runs_->cts());
    518   stream_buf->SetDecodeTimestamp(runs_->dts());
    519 
    520   DVLOG(3) << "Pushing frame: aud=" << audio
    521            << ", key=" << runs_->is_keyframe()
    522            << ", dur=" << runs_->duration().InMilliseconds()
    523            << ", dts=" << runs_->dts().InMilliseconds()
    524            << ", cts=" << runs_->cts().InMilliseconds()
    525            << ", size=" << runs_->sample_size();
    526 
    527   if (audio) {
    528     audio_buffers->push_back(stream_buf);
    529   } else {
    530     video_buffers->push_back(stream_buf);
    531   }
    532 
    533   runs_->AdvanceSample();
    534   return true;
    535 }
    536 
    537 bool MP4StreamParser::SendAndFlushSamples(BufferQueue* audio_buffers,
    538                                           BufferQueue* video_buffers) {
    539   if (audio_buffers->empty() && video_buffers->empty())
    540     return true;
    541 
    542   bool success = new_buffers_cb_.Run(*audio_buffers, *video_buffers);
    543   audio_buffers->clear();
    544   video_buffers->clear();
    545   return success;
    546 }
    547 
    548 bool MP4StreamParser::ReadAndDiscardMDATsUntil(const int64 offset) {
    549   bool err = false;
    550   while (mdat_tail_ < offset) {
    551     const uint8* buf;
    552     int size;
    553     queue_.PeekAt(mdat_tail_, &buf, &size);
    554 
    555     FourCC type;
    556     int box_sz;
    557     if (!BoxReader::StartTopLevelBox(buf, size, log_cb_,
    558                                      &type, &box_sz, &err))
    559       break;
    560 
    561     if (type != FOURCC_MDAT) {
    562       MEDIA_LOG(log_cb_) << "Unexpected box type while parsing MDATs: "
    563                          << FourCCToString(type);
    564     }
    565     mdat_tail_ += box_sz;
    566   }
    567   queue_.Trim(std::min(mdat_tail_, offset));
    568   return !err;
    569 }
    570 
    571 void MP4StreamParser::ChangeState(State new_state) {
    572   DVLOG(2) << "Changing state: " << new_state;
    573   state_ = new_state;
    574 }
    575 
    576 }  // namespace mp4
    577 }  // namespace media
    578