1 // Copyright 2014 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "media/formats/mp4/track_run_iterator.h" 6 7 #include <algorithm> 8 9 #include "media/base/buffers.h" 10 #include "media/formats/mp4/rcheck.h" 11 #include "media/formats/mp4/sample_to_group_iterator.h" 12 13 namespace media { 14 namespace mp4 { 15 16 struct SampleInfo { 17 int size; 18 int duration; 19 int cts_offset; 20 bool is_keyframe; 21 bool is_random_access_point; 22 uint32 cenc_group_description_index; 23 }; 24 25 struct TrackRunInfo { 26 uint32 track_id; 27 std::vector<SampleInfo> samples; 28 int64 timescale; 29 int64 start_dts; 30 int64 sample_start_offset; 31 32 bool is_audio; 33 const AudioSampleEntry* audio_description; 34 const VideoSampleEntry* video_description; 35 36 int64 aux_info_start_offset; // Only valid if aux_info_total_size > 0. 37 int aux_info_default_size; 38 std::vector<uint8> aux_info_sizes; // Populated if default_size == 0. 39 int aux_info_total_size; 40 41 std::vector<CencSampleEncryptionInfoEntry> sample_encryption_info; 42 43 TrackRunInfo(); 44 ~TrackRunInfo(); 45 }; 46 47 TrackRunInfo::TrackRunInfo() 48 : track_id(0), 49 timescale(-1), 50 start_dts(-1), 51 sample_start_offset(-1), 52 is_audio(false), 53 aux_info_start_offset(-1), 54 aux_info_default_size(-1), 55 aux_info_total_size(-1) { 56 } 57 TrackRunInfo::~TrackRunInfo() {} 58 59 base::TimeDelta TimeDeltaFromRational(int64 numer, int64 denom) { 60 // To avoid overflow, split the following calculation: 61 // (numer * base::Time::kMicrosecondsPerSecond) / denom 62 // into: 63 // (numer / denom) * base::Time::kMicrosecondsPerSecond + 64 // ((numer % denom) * base::Time::kMicrosecondsPerSecond) / denom 65 int64 a = numer / denom; 66 DCHECK_LE((a > 0 ? a : -a), kint64max / base::Time::kMicrosecondsPerSecond); 67 int64 timea_in_us = a * base::Time::kMicrosecondsPerSecond; 68 69 int64 b = numer % denom; 70 DCHECK_LE((b > 0 ? b : -b), kint64max / base::Time::kMicrosecondsPerSecond); 71 int64 timeb_in_us = (b * base::Time::kMicrosecondsPerSecond) / denom; 72 73 DCHECK((timeb_in_us < 0) || (timea_in_us <= kint64max - timeb_in_us)); 74 DCHECK((timeb_in_us > 0) || (timea_in_us >= kint64min - timeb_in_us)); 75 return base::TimeDelta::FromMicroseconds(timea_in_us + timeb_in_us); 76 } 77 78 DecodeTimestamp DecodeTimestampFromRational(int64 numer, int64 denom) { 79 return DecodeTimestamp::FromPresentationTime( 80 TimeDeltaFromRational(numer, denom)); 81 } 82 83 TrackRunIterator::TrackRunIterator(const Movie* moov, 84 const LogCB& log_cb) 85 : moov_(moov), log_cb_(log_cb), sample_offset_(0) { 86 CHECK(moov); 87 } 88 89 TrackRunIterator::~TrackRunIterator() {} 90 91 static bool PopulateSampleInfo(const TrackExtends& trex, 92 const TrackFragmentHeader& tfhd, 93 const TrackFragmentRun& trun, 94 const int64 edit_list_offset, 95 const uint32 i, 96 SampleInfo* sample_info, 97 const SampleDependsOn sdtp_sample_depends_on, 98 const LogCB& log_cb) { 99 if (i < trun.sample_sizes.size()) { 100 sample_info->size = trun.sample_sizes[i]; 101 } else if (tfhd.default_sample_size > 0) { 102 sample_info->size = tfhd.default_sample_size; 103 } else { 104 sample_info->size = trex.default_sample_size; 105 } 106 107 if (i < trun.sample_durations.size()) { 108 sample_info->duration = trun.sample_durations[i]; 109 } else if (tfhd.default_sample_duration > 0) { 110 sample_info->duration = tfhd.default_sample_duration; 111 } else { 112 sample_info->duration = trex.default_sample_duration; 113 } 114 115 if (i < trun.sample_composition_time_offsets.size()) { 116 sample_info->cts_offset = trun.sample_composition_time_offsets[i]; 117 } else { 118 sample_info->cts_offset = 0; 119 } 120 sample_info->cts_offset += edit_list_offset; 121 122 uint32 flags; 123 if (i < trun.sample_flags.size()) { 124 flags = trun.sample_flags[i]; 125 } else if (tfhd.has_default_sample_flags) { 126 flags = tfhd.default_sample_flags; 127 } else { 128 flags = trex.default_sample_flags; 129 } 130 131 SampleDependsOn sample_depends_on = 132 static_cast<SampleDependsOn>((flags >> 24) & 0x3); 133 134 if (sample_depends_on == kSampleDependsOnUnknown) 135 sample_depends_on = sdtp_sample_depends_on; 136 137 // ISO/IEC 14496-12 Section 8.8.3.1 : The negation of |sample_is_sync_sample| 138 // provides the same information as the sync sample table [8.6.2]. When 139 // |sample_is_sync_sample| is true for a sample, it is the same as if the 140 // sample were not in a movie fragment and marked with an entry in the sync 141 // sample table (or, if all samples are sync samples, the sync sample table 142 // were absent). 143 bool sample_is_sync_sample = !(flags & kSampleIsNonSyncSample); 144 sample_info->is_random_access_point = sample_is_sync_sample; 145 146 switch (sample_depends_on) { 147 case kSampleDependsOnUnknown: 148 sample_info->is_keyframe = sample_is_sync_sample; 149 break; 150 151 case kSampleDependsOnOthers: 152 sample_info->is_keyframe = false; 153 break; 154 155 case kSampleDependsOnNoOther: 156 sample_info->is_keyframe = true; 157 break; 158 159 case kSampleDependsOnReserved: 160 MEDIA_LOG(log_cb) << "Reserved value used in sample dependency info."; 161 return false; 162 } 163 return true; 164 } 165 166 // In well-structured encrypted media, each track run will be immediately 167 // preceded by its auxiliary information; this is the only optimal storage 168 // pattern in terms of minimum number of bytes from a serial stream needed to 169 // begin playback. It also allows us to optimize caching on memory-constrained 170 // architectures, because we can cache the relatively small auxiliary 171 // information for an entire run and then discard data from the input stream, 172 // instead of retaining the entire 'mdat' box. 173 // 174 // We optimize for this situation (with no loss of generality) by sorting track 175 // runs during iteration in order of their first data offset (either sample data 176 // or auxiliary data). 177 class CompareMinTrackRunDataOffset { 178 public: 179 bool operator()(const TrackRunInfo& a, const TrackRunInfo& b) { 180 int64 a_aux = a.aux_info_total_size ? a.aux_info_start_offset : kint64max; 181 int64 b_aux = b.aux_info_total_size ? b.aux_info_start_offset : kint64max; 182 183 int64 a_lesser = std::min(a_aux, a.sample_start_offset); 184 int64 a_greater = std::max(a_aux, a.sample_start_offset); 185 int64 b_lesser = std::min(b_aux, b.sample_start_offset); 186 int64 b_greater = std::max(b_aux, b.sample_start_offset); 187 188 if (a_lesser == b_lesser) return a_greater < b_greater; 189 return a_lesser < b_lesser; 190 } 191 }; 192 193 bool TrackRunIterator::Init(const MovieFragment& moof) { 194 runs_.clear(); 195 196 for (size_t i = 0; i < moof.tracks.size(); i++) { 197 const TrackFragment& traf = moof.tracks[i]; 198 199 const Track* trak = NULL; 200 for (size_t t = 0; t < moov_->tracks.size(); t++) { 201 if (moov_->tracks[t].header.track_id == traf.header.track_id) 202 trak = &moov_->tracks[t]; 203 } 204 RCHECK(trak); 205 206 const TrackExtends* trex = NULL; 207 for (size_t t = 0; t < moov_->extends.tracks.size(); t++) { 208 if (moov_->extends.tracks[t].track_id == traf.header.track_id) 209 trex = &moov_->extends.tracks[t]; 210 } 211 RCHECK(trex); 212 213 const SampleDescription& stsd = 214 trak->media.information.sample_table.description; 215 if (stsd.type != kAudio && stsd.type != kVideo) { 216 DVLOG(1) << "Skipping unhandled track type"; 217 continue; 218 } 219 size_t desc_idx = traf.header.sample_description_index; 220 if (!desc_idx) desc_idx = trex->default_sample_description_index; 221 RCHECK(desc_idx > 0); // Descriptions are one-indexed in the file 222 desc_idx -= 1; 223 224 // Process edit list to remove CTS offset introduced in the presence of 225 // B-frames (those that contain a single edit with a nonnegative media 226 // time). Other uses of edit lists are not supported, as they are 227 // both uncommon and better served by higher-level protocols. 228 int64 edit_list_offset = 0; 229 const std::vector<EditListEntry>& edits = trak->edit.list.edits; 230 if (!edits.empty()) { 231 if (edits.size() > 1) 232 DVLOG(1) << "Multi-entry edit box detected; some components ignored."; 233 234 if (edits[0].media_time < 0) { 235 DVLOG(1) << "Empty edit list entry ignored."; 236 } else { 237 edit_list_offset = -edits[0].media_time; 238 } 239 } 240 241 SampleToGroupIterator sample_to_group_itr(traf.sample_to_group); 242 bool is_sample_to_group_valid = sample_to_group_itr.IsValid(); 243 244 int64 run_start_dts = traf.decode_time.decode_time; 245 int sample_count_sum = 0; 246 for (size_t j = 0; j < traf.runs.size(); j++) { 247 const TrackFragmentRun& trun = traf.runs[j]; 248 TrackRunInfo tri; 249 tri.track_id = traf.header.track_id; 250 tri.timescale = trak->media.header.timescale; 251 tri.start_dts = run_start_dts; 252 tri.sample_start_offset = trun.data_offset; 253 tri.sample_encryption_info = traf.sample_group_description.entries; 254 255 tri.is_audio = (stsd.type == kAudio); 256 if (tri.is_audio) { 257 RCHECK(!stsd.audio_entries.empty()); 258 if (desc_idx > stsd.audio_entries.size()) 259 desc_idx = 0; 260 tri.audio_description = &stsd.audio_entries[desc_idx]; 261 } else { 262 RCHECK(!stsd.video_entries.empty()); 263 if (desc_idx > stsd.video_entries.size()) 264 desc_idx = 0; 265 tri.video_description = &stsd.video_entries[desc_idx]; 266 } 267 268 // Collect information from the auxiliary_offset entry with the same index 269 // in the 'saiz' container as the current run's index in the 'trun' 270 // container, if it is present. 271 if (traf.auxiliary_offset.offsets.size() > j) { 272 // There should be an auxiliary info entry corresponding to each sample 273 // in the auxiliary offset entry's corresponding track run. 274 RCHECK(traf.auxiliary_size.sample_count >= 275 sample_count_sum + trun.sample_count); 276 tri.aux_info_start_offset = traf.auxiliary_offset.offsets[j]; 277 tri.aux_info_default_size = 278 traf.auxiliary_size.default_sample_info_size; 279 if (tri.aux_info_default_size == 0) { 280 const std::vector<uint8>& sizes = 281 traf.auxiliary_size.sample_info_sizes; 282 tri.aux_info_sizes.insert(tri.aux_info_sizes.begin(), 283 sizes.begin() + sample_count_sum, 284 sizes.begin() + sample_count_sum + trun.sample_count); 285 } 286 287 // If the default info size is positive, find the total size of the aux 288 // info block from it, otherwise sum over the individual sizes of each 289 // aux info entry in the aux_offset entry. 290 if (tri.aux_info_default_size) { 291 tri.aux_info_total_size = 292 tri.aux_info_default_size * trun.sample_count; 293 } else { 294 tri.aux_info_total_size = 0; 295 for (size_t k = 0; k < trun.sample_count; k++) { 296 tri.aux_info_total_size += tri.aux_info_sizes[k]; 297 } 298 } 299 } else { 300 tri.aux_info_start_offset = -1; 301 tri.aux_info_total_size = 0; 302 } 303 304 tri.samples.resize(trun.sample_count); 305 for (size_t k = 0; k < trun.sample_count; k++) { 306 if (!PopulateSampleInfo(*trex, traf.header, trun, edit_list_offset, 307 k, &tri.samples[k], 308 traf.sdtp.sample_depends_on(k), 309 log_cb_)) { 310 return false; 311 } 312 313 run_start_dts += tri.samples[k].duration; 314 315 if (!is_sample_to_group_valid) { 316 // Set group description index to 0 to read encryption information 317 // from TrackEncryption Box. 318 tri.samples[k].cenc_group_description_index = 0; 319 continue; 320 } 321 322 // ISO-14496-12 Section 8.9.2.3 and 8.9.4 : group description index 323 // (1) ranges from 1 to the number of sample group entries in the track 324 // level SampleGroupDescription Box, or (2) takes the value 0 to 325 // indicate that this sample is a member of no group, in this case, the 326 // sample is associated with the default values specified in 327 // TrackEncryption Box, or (3) starts at 0x10001, i.e. the index value 328 // 1, with the value 1 in the top 16 bits, to reference fragment-local 329 // SampleGroupDescription Box. 330 // Case (1) is not supported currently. We might not need it either as 331 // the same functionality can be better achieved using (2). 332 uint32 index = sample_to_group_itr.group_description_index(); 333 if (index >= SampleToGroupEntry::kFragmentGroupDescriptionIndexBase) { 334 index -= SampleToGroupEntry::kFragmentGroupDescriptionIndexBase; 335 RCHECK(index != 0 && index <= tri.sample_encryption_info.size()); 336 } else if (index != 0) { 337 NOTIMPLEMENTED() << "'sgpd' box in 'moov' is not supported."; 338 return false; 339 } 340 tri.samples[k].cenc_group_description_index = index; 341 is_sample_to_group_valid = sample_to_group_itr.Advance(); 342 } 343 runs_.push_back(tri); 344 sample_count_sum += trun.sample_count; 345 } 346 347 // We should have iterated through all samples in SampleToGroup Box. 348 RCHECK(!sample_to_group_itr.IsValid()); 349 } 350 351 std::sort(runs_.begin(), runs_.end(), CompareMinTrackRunDataOffset()); 352 run_itr_ = runs_.begin(); 353 ResetRun(); 354 return true; 355 } 356 357 void TrackRunIterator::AdvanceRun() { 358 ++run_itr_; 359 ResetRun(); 360 } 361 362 void TrackRunIterator::ResetRun() { 363 if (!IsRunValid()) return; 364 sample_dts_ = run_itr_->start_dts; 365 sample_offset_ = run_itr_->sample_start_offset; 366 sample_itr_ = run_itr_->samples.begin(); 367 cenc_info_.clear(); 368 } 369 370 void TrackRunIterator::AdvanceSample() { 371 DCHECK(IsSampleValid()); 372 sample_dts_ += sample_itr_->duration; 373 sample_offset_ += sample_itr_->size; 374 ++sample_itr_; 375 } 376 377 // This implementation only indicates a need for caching if CENC auxiliary 378 // info is available in the stream. 379 bool TrackRunIterator::AuxInfoNeedsToBeCached() { 380 DCHECK(IsRunValid()); 381 return aux_info_size() > 0 && cenc_info_.size() == 0; 382 } 383 384 // This implementation currently only caches CENC auxiliary info. 385 bool TrackRunIterator::CacheAuxInfo(const uint8* buf, int buf_size) { 386 RCHECK(AuxInfoNeedsToBeCached() && buf_size >= aux_info_size()); 387 388 cenc_info_.resize(run_itr_->samples.size()); 389 int64 pos = 0; 390 for (size_t i = 0; i < run_itr_->samples.size(); i++) { 391 int info_size = run_itr_->aux_info_default_size; 392 if (!info_size) 393 info_size = run_itr_->aux_info_sizes[i]; 394 395 if (IsSampleEncrypted(i)) { 396 BufferReader reader(buf + pos, info_size); 397 RCHECK(cenc_info_[i].Parse(GetIvSize(i), &reader)); 398 } 399 pos += info_size; 400 } 401 402 return true; 403 } 404 405 bool TrackRunIterator::IsRunValid() const { 406 return run_itr_ != runs_.end(); 407 } 408 409 bool TrackRunIterator::IsSampleValid() const { 410 return IsRunValid() && (sample_itr_ != run_itr_->samples.end()); 411 } 412 413 // Because tracks are in sorted order and auxiliary information is cached when 414 // returning samples, it is guaranteed that no data will be required before the 415 // lesser of the minimum data offset of this track and the next in sequence. 416 // (The stronger condition - that no data is required before the minimum data 417 // offset of this track alone - is not guaranteed, because the BMFF spec does 418 // not have any inter-run ordering restrictions.) 419 int64 TrackRunIterator::GetMaxClearOffset() { 420 int64 offset = kint64max; 421 422 if (IsSampleValid()) { 423 offset = std::min(offset, sample_offset_); 424 if (AuxInfoNeedsToBeCached()) 425 offset = std::min(offset, aux_info_offset()); 426 } 427 if (run_itr_ != runs_.end()) { 428 std::vector<TrackRunInfo>::const_iterator next_run = run_itr_ + 1; 429 if (next_run != runs_.end()) { 430 offset = std::min(offset, next_run->sample_start_offset); 431 if (next_run->aux_info_total_size) 432 offset = std::min(offset, next_run->aux_info_start_offset); 433 } 434 } 435 if (offset == kint64max) return 0; 436 return offset; 437 } 438 439 uint32 TrackRunIterator::track_id() const { 440 DCHECK(IsRunValid()); 441 return run_itr_->track_id; 442 } 443 444 bool TrackRunIterator::is_encrypted() const { 445 DCHECK(IsSampleValid()); 446 return IsSampleEncrypted(sample_itr_ - run_itr_->samples.begin()); 447 } 448 449 int64 TrackRunIterator::aux_info_offset() const { 450 return run_itr_->aux_info_start_offset; 451 } 452 453 int TrackRunIterator::aux_info_size() const { 454 return run_itr_->aux_info_total_size; 455 } 456 457 bool TrackRunIterator::is_audio() const { 458 DCHECK(IsRunValid()); 459 return run_itr_->is_audio; 460 } 461 462 const AudioSampleEntry& TrackRunIterator::audio_description() const { 463 DCHECK(is_audio()); 464 DCHECK(run_itr_->audio_description); 465 return *run_itr_->audio_description; 466 } 467 468 const VideoSampleEntry& TrackRunIterator::video_description() const { 469 DCHECK(!is_audio()); 470 DCHECK(run_itr_->video_description); 471 return *run_itr_->video_description; 472 } 473 474 int64 TrackRunIterator::sample_offset() const { 475 DCHECK(IsSampleValid()); 476 return sample_offset_; 477 } 478 479 int TrackRunIterator::sample_size() const { 480 DCHECK(IsSampleValid()); 481 return sample_itr_->size; 482 } 483 484 DecodeTimestamp TrackRunIterator::dts() const { 485 DCHECK(IsSampleValid()); 486 return DecodeTimestampFromRational(sample_dts_, run_itr_->timescale); 487 } 488 489 base::TimeDelta TrackRunIterator::cts() const { 490 DCHECK(IsSampleValid()); 491 return TimeDeltaFromRational(sample_dts_ + sample_itr_->cts_offset, 492 run_itr_->timescale); 493 } 494 495 base::TimeDelta TrackRunIterator::duration() const { 496 DCHECK(IsSampleValid()); 497 return TimeDeltaFromRational(sample_itr_->duration, run_itr_->timescale); 498 } 499 500 bool TrackRunIterator::is_keyframe() const { 501 DCHECK(IsSampleValid()); 502 return sample_itr_->is_keyframe; 503 } 504 505 bool TrackRunIterator::is_random_access_point() const { 506 DCHECK(IsSampleValid()); 507 return sample_itr_->is_random_access_point; 508 } 509 510 const TrackEncryption& TrackRunIterator::track_encryption() const { 511 if (is_audio()) 512 return audio_description().sinf.info.track_encryption; 513 return video_description().sinf.info.track_encryption; 514 } 515 516 scoped_ptr<DecryptConfig> TrackRunIterator::GetDecryptConfig() { 517 DCHECK(is_encrypted()); 518 519 if (cenc_info_.empty()) { 520 DCHECK_EQ(0, aux_info_size()); 521 MEDIA_LOG(log_cb_) << "Aux Info is not available."; 522 return scoped_ptr<DecryptConfig>(); 523 } 524 525 size_t sample_idx = sample_itr_ - run_itr_->samples.begin(); 526 DCHECK_LT(sample_idx, cenc_info_.size()); 527 const FrameCENCInfo& cenc_info = cenc_info_[sample_idx]; 528 529 size_t total_size = 0; 530 if (!cenc_info.subsamples.empty() && 531 (!cenc_info.GetTotalSizeOfSubsamples(&total_size) || 532 total_size != static_cast<size_t>(sample_size()))) { 533 MEDIA_LOG(log_cb_) << "Incorrect CENC subsample size."; 534 return scoped_ptr<DecryptConfig>(); 535 } 536 537 const std::vector<uint8>& kid = GetKeyId(sample_idx); 538 return scoped_ptr<DecryptConfig>(new DecryptConfig( 539 std::string(reinterpret_cast<const char*>(&kid[0]), kid.size()), 540 std::string(reinterpret_cast<const char*>(cenc_info.iv), 541 arraysize(cenc_info.iv)), 542 cenc_info.subsamples)); 543 } 544 545 uint32 TrackRunIterator::GetGroupDescriptionIndex(uint32 sample_index) const { 546 DCHECK(IsRunValid()); 547 DCHECK_LT(sample_index, run_itr_->samples.size()); 548 return run_itr_->samples[sample_index].cenc_group_description_index; 549 } 550 551 const CencSampleEncryptionInfoEntry& 552 TrackRunIterator::GetSampleEncryptionInfoEntry( 553 uint32 group_description_index) const { 554 DCHECK(IsRunValid()); 555 DCHECK_NE(group_description_index, 0u); 556 DCHECK_LE(group_description_index, run_itr_->sample_encryption_info.size()); 557 // |group_description_index| is 1-based. Subtract by 1 to index the vector. 558 return run_itr_->sample_encryption_info[group_description_index - 1]; 559 } 560 561 bool TrackRunIterator::IsSampleEncrypted(size_t sample_index) const { 562 uint32 index = GetGroupDescriptionIndex(sample_index); 563 return (index == 0) ? track_encryption().is_encrypted 564 : GetSampleEncryptionInfoEntry(index).is_encrypted; 565 } 566 567 const std::vector<uint8>& TrackRunIterator::GetKeyId( 568 size_t sample_index) const { 569 uint32 index = GetGroupDescriptionIndex(sample_index); 570 return (index == 0) ? track_encryption().default_kid 571 : GetSampleEncryptionInfoEntry(index).key_id; 572 } 573 574 uint8 TrackRunIterator::GetIvSize(size_t sample_index) const { 575 uint32 index = GetGroupDescriptionIndex(sample_index); 576 return (index == 0) ? track_encryption().default_iv_size 577 : GetSampleEncryptionInfoEntry(index).iv_size; 578 } 579 580 } // namespace mp4 581 } // namespace media 582