1 // Copyright 2014 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "media/formats/mp2t/es_parser_h264.h" 6 7 #include "base/basictypes.h" 8 #include "base/logging.h" 9 #include "base/numerics/safe_conversions.h" 10 #include "media/base/buffers.h" 11 #include "media/base/stream_parser_buffer.h" 12 #include "media/base/video_frame.h" 13 #include "media/filters/h264_parser.h" 14 #include "media/formats/common/offset_byte_queue.h" 15 #include "media/formats/mp2t/mp2t_common.h" 16 #include "ui/gfx/rect.h" 17 #include "ui/gfx/size.h" 18 19 namespace media { 20 namespace mp2t { 21 22 // An AUD NALU is at least 4 bytes: 23 // 3 bytes for the start code + 1 byte for the NALU type. 24 const int kMinAUDSize = 4; 25 26 EsParserH264::EsParserH264( 27 const NewVideoConfigCB& new_video_config_cb, 28 const EmitBufferCB& emit_buffer_cb) 29 : new_video_config_cb_(new_video_config_cb), 30 emit_buffer_cb_(emit_buffer_cb), 31 es_queue_(new media::OffsetByteQueue()), 32 h264_parser_(new H264Parser()), 33 current_access_unit_pos_(0), 34 next_access_unit_pos_(0) { 35 } 36 37 EsParserH264::~EsParserH264() { 38 } 39 40 bool EsParserH264::Parse(const uint8* buf, int size, 41 base::TimeDelta pts, 42 base::TimeDelta dts) { 43 // Note: Parse is invoked each time a PES packet has been reassembled. 44 // Unfortunately, a PES packet does not necessarily map 45 // to an h264 access unit, although the HLS recommendation is to use one PES 46 // for each access unit (but this is just a recommendation and some streams 47 // do not comply with this recommendation). 48 49 // HLS recommendation: "In AVC video, you should have both a DTS and a 50 // PTS in each PES header". 51 // However, some streams do not comply with this recommendation. 52 DVLOG_IF(1, pts == kNoTimestamp()) << "Each video PES should have a PTS"; 53 if (pts != kNoTimestamp()) { 54 TimingDesc timing_desc; 55 timing_desc.pts = pts; 56 timing_desc.dts = (dts != kNoTimestamp()) ? dts : pts; 57 58 // Link the end of the byte queue with the incoming timing descriptor. 59 timing_desc_list_.push_back( 60 std::pair<int64, TimingDesc>(es_queue_->tail(), timing_desc)); 61 } 62 63 // Add the incoming bytes to the ES queue. 64 es_queue_->Push(buf, size); 65 return ParseInternal(); 66 } 67 68 void EsParserH264::Flush() { 69 DVLOG(1) << "EsParserH264::Flush"; 70 if (!FindAUD(¤t_access_unit_pos_)) 71 return; 72 73 // Simulate an additional AUD to force emitting the last access unit 74 // which is assumed to be complete at this point. 75 uint8 aud[] = { 0x00, 0x00, 0x01, 0x09 }; 76 es_queue_->Push(aud, sizeof(aud)); 77 ParseInternal(); 78 } 79 80 void EsParserH264::Reset() { 81 DVLOG(1) << "EsParserH264::Reset"; 82 es_queue_.reset(new media::OffsetByteQueue()); 83 h264_parser_.reset(new H264Parser()); 84 current_access_unit_pos_ = 0; 85 next_access_unit_pos_ = 0; 86 timing_desc_list_.clear(); 87 last_video_decoder_config_ = VideoDecoderConfig(); 88 } 89 90 bool EsParserH264::FindAUD(int64* stream_pos) { 91 while (true) { 92 const uint8* es; 93 int size; 94 es_queue_->PeekAt(*stream_pos, &es, &size); 95 96 // Find a start code and move the stream to the start code parser position. 97 off_t start_code_offset; 98 off_t start_code_size; 99 bool start_code_found = H264Parser::FindStartCode( 100 es, size, &start_code_offset, &start_code_size); 101 *stream_pos += start_code_offset; 102 103 // No H264 start code found or NALU type not available yet. 104 if (!start_code_found || start_code_offset + start_code_size >= size) 105 return false; 106 107 // Exit the parser loop when an AUD is found. 108 // Note: NALU header for an AUD: 109 // - nal_ref_idc must be 0 110 // - nal_unit_type must be H264NALU::kAUD 111 if (es[start_code_offset + start_code_size] == H264NALU::kAUD) 112 break; 113 114 // The current NALU is not an AUD, skip the start code 115 // and continue parsing the stream. 116 *stream_pos += start_code_size; 117 } 118 119 return true; 120 } 121 122 bool EsParserH264::ParseInternal() { 123 DCHECK_LE(es_queue_->head(), current_access_unit_pos_); 124 DCHECK_LE(current_access_unit_pos_, next_access_unit_pos_); 125 DCHECK_LE(next_access_unit_pos_, es_queue_->tail()); 126 127 // Find the next AUD located at or after |current_access_unit_pos_|. This is 128 // needed since initially |current_access_unit_pos_| might not point to 129 // an AUD. 130 // Discard all the data before the updated |current_access_unit_pos_| 131 // since it won't be used again. 132 bool aud_found = FindAUD(¤t_access_unit_pos_); 133 es_queue_->Trim(current_access_unit_pos_); 134 if (next_access_unit_pos_ < current_access_unit_pos_) 135 next_access_unit_pos_ = current_access_unit_pos_; 136 137 // Resume parsing later if no AUD was found. 138 if (!aud_found) 139 return true; 140 141 // Find the next AUD to make sure we have a complete access unit. 142 if (next_access_unit_pos_ < current_access_unit_pos_ + kMinAUDSize) { 143 next_access_unit_pos_ = current_access_unit_pos_ + kMinAUDSize; 144 DCHECK_LE(next_access_unit_pos_, es_queue_->tail()); 145 } 146 if (!FindAUD(&next_access_unit_pos_)) 147 return true; 148 149 // At this point, we know we have a full access unit. 150 bool is_key_frame = false; 151 int pps_id_for_access_unit = -1; 152 153 const uint8* es; 154 int size; 155 es_queue_->PeekAt(current_access_unit_pos_, &es, &size); 156 int access_unit_size = base::checked_cast<int, int64>( 157 next_access_unit_pos_ - current_access_unit_pos_); 158 DCHECK_LE(access_unit_size, size); 159 h264_parser_->SetStream(es, access_unit_size); 160 161 while (true) { 162 bool is_eos = false; 163 H264NALU nalu; 164 switch (h264_parser_->AdvanceToNextNALU(&nalu)) { 165 case H264Parser::kOk: 166 break; 167 case H264Parser::kInvalidStream: 168 case H264Parser::kUnsupportedStream: 169 return false; 170 case H264Parser::kEOStream: 171 is_eos = true; 172 break; 173 } 174 if (is_eos) 175 break; 176 177 switch (nalu.nal_unit_type) { 178 case H264NALU::kAUD: { 179 DVLOG(LOG_LEVEL_ES) << "NALU: AUD"; 180 break; 181 } 182 case H264NALU::kSPS: { 183 DVLOG(LOG_LEVEL_ES) << "NALU: SPS"; 184 int sps_id; 185 if (h264_parser_->ParseSPS(&sps_id) != H264Parser::kOk) 186 return false; 187 break; 188 } 189 case H264NALU::kPPS: { 190 DVLOG(LOG_LEVEL_ES) << "NALU: PPS"; 191 int pps_id; 192 if (h264_parser_->ParsePPS(&pps_id) != H264Parser::kOk) 193 return false; 194 break; 195 } 196 case H264NALU::kIDRSlice: 197 case H264NALU::kNonIDRSlice: { 198 is_key_frame = (nalu.nal_unit_type == H264NALU::kIDRSlice); 199 DVLOG(LOG_LEVEL_ES) << "NALU: slice IDR=" << is_key_frame; 200 H264SliceHeader shdr; 201 if (h264_parser_->ParseSliceHeader(nalu, &shdr) != H264Parser::kOk) { 202 // Only accept an invalid SPS/PPS at the beginning when the stream 203 // does not necessarily start with an SPS/PPS/IDR. 204 // TODO(damienv): Should be able to differentiate a missing SPS/PPS 205 // from a slice header parsing error. 206 if (last_video_decoder_config_.IsValidConfig()) 207 return false; 208 } else { 209 pps_id_for_access_unit = shdr.pic_parameter_set_id; 210 } 211 break; 212 } 213 default: { 214 DVLOG(LOG_LEVEL_ES) << "NALU: " << nalu.nal_unit_type; 215 } 216 } 217 } 218 219 // Emit a frame and move the stream to the next AUD position. 220 RCHECK(EmitFrame(current_access_unit_pos_, access_unit_size, 221 is_key_frame, pps_id_for_access_unit)); 222 current_access_unit_pos_ = next_access_unit_pos_; 223 es_queue_->Trim(current_access_unit_pos_); 224 225 return true; 226 } 227 228 bool EsParserH264::EmitFrame(int64 access_unit_pos, int access_unit_size, 229 bool is_key_frame, int pps_id) { 230 // Get the access unit timing info. 231 TimingDesc current_timing_desc = {kNoTimestamp(), kNoTimestamp()}; 232 while (!timing_desc_list_.empty() && 233 timing_desc_list_.front().first <= access_unit_pos) { 234 current_timing_desc = timing_desc_list_.front().second; 235 timing_desc_list_.pop_front(); 236 } 237 if (current_timing_desc.pts == kNoTimestamp()) 238 return false; 239 240 // Update the video decoder configuration if needed. 241 const H264PPS* pps = h264_parser_->GetPPS(pps_id); 242 if (!pps) { 243 // Only accept an invalid PPS at the beginning when the stream 244 // does not necessarily start with an SPS/PPS/IDR. 245 // In this case, the initial frames are conveyed to the upper layer with 246 // an invalid VideoDecoderConfig and it's up to the upper layer 247 // to process this kind of frame accordingly. 248 if (last_video_decoder_config_.IsValidConfig()) 249 return false; 250 } else { 251 const H264SPS* sps = h264_parser_->GetSPS(pps->seq_parameter_set_id); 252 if (!sps) 253 return false; 254 RCHECK(UpdateVideoDecoderConfig(sps)); 255 } 256 257 // Emit a frame. 258 DVLOG(LOG_LEVEL_ES) << "Emit frame: stream_pos=" << current_access_unit_pos_ 259 << " size=" << access_unit_size; 260 int es_size; 261 const uint8* es; 262 es_queue_->PeekAt(current_access_unit_pos_, &es, &es_size); 263 CHECK_GE(es_size, access_unit_size); 264 265 // TODO(wolenetz/acolwell): Validate and use a common cross-parser TrackId 266 // type and allow multiple video tracks. See https://crbug.com/341581. 267 scoped_refptr<StreamParserBuffer> stream_parser_buffer = 268 StreamParserBuffer::CopyFrom( 269 es, 270 access_unit_size, 271 is_key_frame, 272 DemuxerStream::VIDEO, 273 0); 274 stream_parser_buffer->SetDecodeTimestamp(current_timing_desc.dts); 275 stream_parser_buffer->set_timestamp(current_timing_desc.pts); 276 emit_buffer_cb_.Run(stream_parser_buffer); 277 return true; 278 } 279 280 bool EsParserH264::UpdateVideoDecoderConfig(const H264SPS* sps) { 281 // Set the SAR to 1 when not specified in the H264 stream. 282 int sar_width = (sps->sar_width == 0) ? 1 : sps->sar_width; 283 int sar_height = (sps->sar_height == 0) ? 1 : sps->sar_height; 284 285 // TODO(damienv): a MAP unit can be either 16 or 32 pixels. 286 // although it's 16 pixels for progressive non MBAFF frames. 287 gfx::Size coded_size((sps->pic_width_in_mbs_minus1 + 1) * 16, 288 (sps->pic_height_in_map_units_minus1 + 1) * 16); 289 gfx::Rect visible_rect( 290 sps->frame_crop_left_offset, 291 sps->frame_crop_top_offset, 292 (coded_size.width() - sps->frame_crop_right_offset) - 293 sps->frame_crop_left_offset, 294 (coded_size.height() - sps->frame_crop_bottom_offset) - 295 sps->frame_crop_top_offset); 296 if (visible_rect.width() <= 0 || visible_rect.height() <= 0) 297 return false; 298 gfx::Size natural_size( 299 (visible_rect.width() * sar_width) / sar_height, 300 visible_rect.height()); 301 if (natural_size.width() == 0) 302 return false; 303 304 VideoDecoderConfig video_decoder_config( 305 kCodecH264, 306 VIDEO_CODEC_PROFILE_UNKNOWN, 307 VideoFrame::YV12, 308 coded_size, 309 visible_rect, 310 natural_size, 311 NULL, 0, 312 false); 313 314 if (!video_decoder_config.Matches(last_video_decoder_config_)) { 315 DVLOG(1) << "Profile IDC: " << sps->profile_idc; 316 DVLOG(1) << "Level IDC: " << sps->level_idc; 317 DVLOG(1) << "Pic width: " << coded_size.width(); 318 DVLOG(1) << "Pic height: " << coded_size.height(); 319 DVLOG(1) << "log2_max_frame_num_minus4: " 320 << sps->log2_max_frame_num_minus4; 321 DVLOG(1) << "SAR: width=" << sps->sar_width 322 << " height=" << sps->sar_height; 323 last_video_decoder_config_ = video_decoder_config; 324 new_video_config_cb_.Run(video_decoder_config); 325 } 326 327 return true; 328 } 329 330 } // namespace mp2t 331 } // namespace media 332 333