1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include <algorithm> 6 #include <limits> 7 8 #include "base/bind.h" 9 #include "base/bind_helpers.h" 10 #include "base/callback_helpers.h" 11 #include "base/macros.h" 12 #include "base/numerics/safe_conversions.h" 13 #include "base/optional.h" 14 #include "base/stl_util.h" 15 #include "h264_decoder.h" 16 17 namespace media { 18 19 H264Decoder::H264Accelerator::H264Accelerator() {} 20 21 H264Decoder::H264Accelerator::~H264Accelerator() {} 22 23 H264Decoder::H264Decoder(H264Accelerator* accelerator) 24 : max_frame_num_(0), 25 max_pic_num_(0), 26 max_long_term_frame_idx_(0), 27 max_num_reorder_frames_(0), 28 accelerator_(accelerator) { 29 DCHECK(accelerator_); 30 Reset(); 31 state_ = kNeedStreamMetadata; 32 } 33 34 H264Decoder::~H264Decoder() {} 35 36 void H264Decoder::Reset() { 37 curr_pic_ = nullptr; 38 curr_nalu_ = nullptr; 39 curr_slice_hdr_ = nullptr; 40 curr_sps_id_ = -1; 41 curr_pps_id_ = -1; 42 43 prev_frame_num_ = -1; 44 prev_ref_frame_num_ = -1; 45 prev_frame_num_offset_ = -1; 46 prev_has_memmgmnt5_ = false; 47 48 prev_ref_has_memmgmnt5_ = false; 49 prev_ref_top_field_order_cnt_ = -1; 50 prev_ref_pic_order_cnt_msb_ = -1; 51 prev_ref_pic_order_cnt_lsb_ = -1; 52 prev_ref_field_ = H264Picture::FIELD_NONE; 53 54 ref_pic_list_p0_.clear(); 55 ref_pic_list_b0_.clear(); 56 ref_pic_list_b1_.clear(); 57 dpb_.Clear(); 58 parser_.Reset(); 59 accelerator_->Reset(); 60 last_output_poc_ = std::numeric_limits<int>::min(); 61 62 // If we are in kDecoding, we can resume without processing an SPS. 63 if (state_ == kDecoding) 64 state_ = kAfterReset; 65 } 66 67 void H264Decoder::PrepareRefPicLists(const H264SliceHeader* slice_hdr) { 68 ConstructReferencePicListsP(slice_hdr); 69 ConstructReferencePicListsB(slice_hdr); 70 } 71 72 bool H264Decoder::ModifyReferencePicLists(const H264SliceHeader* slice_hdr, 73 H264Picture::Vector* ref_pic_list0, 74 H264Picture::Vector* ref_pic_list1) { 75 ref_pic_list0->clear(); 76 ref_pic_list1->clear(); 77 78 // Fill reference picture lists for B and S/SP slices. 79 if (slice_hdr->IsPSlice() || slice_hdr->IsSPSlice()) { 80 *ref_pic_list0 = ref_pic_list_p0_; 81 return ModifyReferencePicList(slice_hdr, 0, ref_pic_list0); 82 } else if (slice_hdr->IsBSlice()) { 83 *ref_pic_list0 = ref_pic_list_b0_; 84 *ref_pic_list1 = ref_pic_list_b1_; 85 return ModifyReferencePicList(slice_hdr, 0, ref_pic_list0) && 86 ModifyReferencePicList(slice_hdr, 1, ref_pic_list1); 87 } 88 89 return true; 90 } 91 92 bool H264Decoder::DecodePicture() { 93 DCHECK(curr_pic_.get()); 94 95 DVLOG(4) << "Decoding POC " << curr_pic_->pic_order_cnt; 96 return accelerator_->SubmitDecode(curr_pic_); 97 } 98 99 bool H264Decoder::InitNonexistingPicture(scoped_refptr<H264Picture> pic, 100 int frame_num) { 101 pic->nonexisting = true; 102 pic->nal_ref_idc = 1; 103 pic->frame_num = pic->pic_num = frame_num; 104 pic->adaptive_ref_pic_marking_mode_flag = false; 105 pic->ref = true; 106 pic->long_term_reference_flag = false; 107 pic->field = H264Picture::FIELD_NONE; 108 109 return CalculatePicOrderCounts(pic); 110 } 111 112 bool H264Decoder::InitCurrPicture(const H264SliceHeader* slice_hdr) { 113 DCHECK(curr_pic_.get()); 114 115 curr_pic_->idr = slice_hdr->idr_pic_flag; 116 if (curr_pic_->idr) 117 curr_pic_->idr_pic_id = slice_hdr->idr_pic_id; 118 119 if (slice_hdr->field_pic_flag) { 120 curr_pic_->field = slice_hdr->bottom_field_flag ? H264Picture::FIELD_BOTTOM 121 : H264Picture::FIELD_TOP; 122 } else { 123 curr_pic_->field = H264Picture::FIELD_NONE; 124 } 125 126 if (curr_pic_->field != H264Picture::FIELD_NONE) { 127 DVLOG(1) << "Interlaced video not supported."; 128 return false; 129 } 130 131 curr_pic_->nal_ref_idc = slice_hdr->nal_ref_idc; 132 curr_pic_->ref = slice_hdr->nal_ref_idc != 0; 133 // This assumes non-interlaced stream. 134 curr_pic_->frame_num = curr_pic_->pic_num = slice_hdr->frame_num; 135 136 DCHECK_NE(curr_sps_id_, -1); 137 const H264SPS* sps = parser_.GetSPS(curr_sps_id_); 138 if (!sps) 139 return false; 140 141 curr_pic_->pic_order_cnt_type = sps->pic_order_cnt_type; 142 switch (curr_pic_->pic_order_cnt_type) { 143 case 0: 144 curr_pic_->pic_order_cnt_lsb = slice_hdr->pic_order_cnt_lsb; 145 curr_pic_->delta_pic_order_cnt_bottom = 146 slice_hdr->delta_pic_order_cnt_bottom; 147 break; 148 149 case 1: 150 curr_pic_->delta_pic_order_cnt0 = slice_hdr->delta_pic_order_cnt0; 151 curr_pic_->delta_pic_order_cnt1 = slice_hdr->delta_pic_order_cnt1; 152 break; 153 154 case 2: 155 break; 156 157 default: 158 NOTREACHED(); 159 return false; 160 } 161 162 if (!CalculatePicOrderCounts(curr_pic_)) 163 return false; 164 165 curr_pic_->long_term_reference_flag = slice_hdr->long_term_reference_flag; 166 curr_pic_->adaptive_ref_pic_marking_mode_flag = 167 slice_hdr->adaptive_ref_pic_marking_mode_flag; 168 169 // If the slice header indicates we will have to perform reference marking 170 // process after this picture is decoded, store required data for that 171 // purpose. 172 if (slice_hdr->adaptive_ref_pic_marking_mode_flag) { 173 static_assert(sizeof(curr_pic_->ref_pic_marking) == 174 sizeof(slice_hdr->ref_pic_marking), 175 "Array sizes of ref pic marking do not match."); 176 memcpy(curr_pic_->ref_pic_marking, slice_hdr->ref_pic_marking, 177 sizeof(curr_pic_->ref_pic_marking)); 178 } 179 180 return true; 181 } 182 183 bool H264Decoder::CalculatePicOrderCounts(scoped_refptr<H264Picture> pic) { 184 const H264SPS* sps = parser_.GetSPS(curr_sps_id_); 185 if (!sps) 186 return false; 187 188 switch (pic->pic_order_cnt_type) { 189 case 0: { 190 // See spec 8.2.1.1. 191 int prev_pic_order_cnt_msb, prev_pic_order_cnt_lsb; 192 193 if (pic->idr) { 194 prev_pic_order_cnt_msb = prev_pic_order_cnt_lsb = 0; 195 } else { 196 if (prev_ref_has_memmgmnt5_) { 197 if (prev_ref_field_ != H264Picture::FIELD_BOTTOM) { 198 prev_pic_order_cnt_msb = 0; 199 prev_pic_order_cnt_lsb = prev_ref_top_field_order_cnt_; 200 } else { 201 prev_pic_order_cnt_msb = 0; 202 prev_pic_order_cnt_lsb = 0; 203 } 204 } else { 205 prev_pic_order_cnt_msb = prev_ref_pic_order_cnt_msb_; 206 prev_pic_order_cnt_lsb = prev_ref_pic_order_cnt_lsb_; 207 } 208 } 209 210 int max_pic_order_cnt_lsb = 211 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4); 212 DCHECK_NE(max_pic_order_cnt_lsb, 0); 213 if ((pic->pic_order_cnt_lsb < prev_pic_order_cnt_lsb) && 214 (prev_pic_order_cnt_lsb - pic->pic_order_cnt_lsb >= 215 max_pic_order_cnt_lsb / 2)) { 216 pic->pic_order_cnt_msb = prev_pic_order_cnt_msb + max_pic_order_cnt_lsb; 217 } else if ((pic->pic_order_cnt_lsb > prev_pic_order_cnt_lsb) && 218 (pic->pic_order_cnt_lsb - prev_pic_order_cnt_lsb > 219 max_pic_order_cnt_lsb / 2)) { 220 pic->pic_order_cnt_msb = prev_pic_order_cnt_msb - max_pic_order_cnt_lsb; 221 } else { 222 pic->pic_order_cnt_msb = prev_pic_order_cnt_msb; 223 } 224 225 if (pic->field != H264Picture::FIELD_BOTTOM) { 226 pic->top_field_order_cnt = 227 pic->pic_order_cnt_msb + pic->pic_order_cnt_lsb; 228 } 229 230 if (pic->field != H264Picture::FIELD_TOP) { 231 if (pic->field == H264Picture::FIELD_NONE) { 232 pic->bottom_field_order_cnt = 233 pic->top_field_order_cnt + pic->delta_pic_order_cnt_bottom; 234 } else { 235 pic->bottom_field_order_cnt = 236 pic->pic_order_cnt_msb + pic->pic_order_cnt_lsb; 237 } 238 } 239 break; 240 } 241 242 case 1: { 243 // See spec 8.2.1.2. 244 if (prev_has_memmgmnt5_) 245 prev_frame_num_offset_ = 0; 246 247 if (pic->idr) 248 pic->frame_num_offset = 0; 249 else if (prev_frame_num_ > pic->frame_num) 250 pic->frame_num_offset = prev_frame_num_offset_ + max_frame_num_; 251 else 252 pic->frame_num_offset = prev_frame_num_offset_; 253 254 int abs_frame_num = 0; 255 if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0) 256 abs_frame_num = pic->frame_num_offset + pic->frame_num; 257 else 258 abs_frame_num = 0; 259 260 if (pic->nal_ref_idc == 0 && abs_frame_num > 0) 261 --abs_frame_num; 262 263 int expected_pic_order_cnt = 0; 264 if (abs_frame_num > 0) { 265 if (sps->num_ref_frames_in_pic_order_cnt_cycle == 0) { 266 DVLOG(1) << "Invalid num_ref_frames_in_pic_order_cnt_cycle " 267 << "in stream"; 268 return false; 269 } 270 271 int pic_order_cnt_cycle_cnt = 272 (abs_frame_num - 1) / sps->num_ref_frames_in_pic_order_cnt_cycle; 273 int frame_num_in_pic_order_cnt_cycle = 274 (abs_frame_num - 1) % sps->num_ref_frames_in_pic_order_cnt_cycle; 275 276 expected_pic_order_cnt = pic_order_cnt_cycle_cnt * 277 sps->expected_delta_per_pic_order_cnt_cycle; 278 // frame_num_in_pic_order_cnt_cycle is verified < 255 in parser 279 for (int i = 0; i <= frame_num_in_pic_order_cnt_cycle; ++i) 280 expected_pic_order_cnt += sps->offset_for_ref_frame[i]; 281 } 282 283 if (!pic->nal_ref_idc) 284 expected_pic_order_cnt += sps->offset_for_non_ref_pic; 285 286 if (pic->field == H264Picture::FIELD_NONE) { 287 pic->top_field_order_cnt = 288 expected_pic_order_cnt + pic->delta_pic_order_cnt0; 289 pic->bottom_field_order_cnt = pic->top_field_order_cnt + 290 sps->offset_for_top_to_bottom_field + 291 pic->delta_pic_order_cnt1; 292 } else if (pic->field != H264Picture::FIELD_BOTTOM) { 293 pic->top_field_order_cnt = 294 expected_pic_order_cnt + pic->delta_pic_order_cnt0; 295 } else { 296 pic->bottom_field_order_cnt = expected_pic_order_cnt + 297 sps->offset_for_top_to_bottom_field + 298 pic->delta_pic_order_cnt0; 299 } 300 break; 301 } 302 303 case 2: { 304 // See spec 8.2.1.3. 305 if (prev_has_memmgmnt5_) 306 prev_frame_num_offset_ = 0; 307 308 if (pic->idr) 309 pic->frame_num_offset = 0; 310 else if (prev_frame_num_ > pic->frame_num) 311 pic->frame_num_offset = prev_frame_num_offset_ + max_frame_num_; 312 else 313 pic->frame_num_offset = prev_frame_num_offset_; 314 315 int temp_pic_order_cnt; 316 if (pic->idr) { 317 temp_pic_order_cnt = 0; 318 } else if (!pic->nal_ref_idc) { 319 temp_pic_order_cnt = 2 * (pic->frame_num_offset + pic->frame_num) - 1; 320 } else { 321 temp_pic_order_cnt = 2 * (pic->frame_num_offset + pic->frame_num); 322 } 323 324 if (pic->field == H264Picture::FIELD_NONE) { 325 pic->top_field_order_cnt = temp_pic_order_cnt; 326 pic->bottom_field_order_cnt = temp_pic_order_cnt; 327 } else if (pic->field == H264Picture::FIELD_BOTTOM) { 328 pic->bottom_field_order_cnt = temp_pic_order_cnt; 329 } else { 330 pic->top_field_order_cnt = temp_pic_order_cnt; 331 } 332 break; 333 } 334 335 default: 336 DVLOG(1) << "Invalid pic_order_cnt_type: " << sps->pic_order_cnt_type; 337 return false; 338 } 339 340 switch (pic->field) { 341 case H264Picture::FIELD_NONE: 342 pic->pic_order_cnt = 343 std::min(pic->top_field_order_cnt, pic->bottom_field_order_cnt); 344 break; 345 case H264Picture::FIELD_TOP: 346 pic->pic_order_cnt = pic->top_field_order_cnt; 347 break; 348 case H264Picture::FIELD_BOTTOM: 349 pic->pic_order_cnt = pic->bottom_field_order_cnt; 350 break; 351 } 352 353 return true; 354 } 355 356 void H264Decoder::UpdatePicNums(int frame_num) { 357 for (auto& pic : dpb_) { 358 if (!pic->ref) 359 continue; 360 361 // 8.2.4.1. Assumes non-interlaced stream. 362 DCHECK_EQ(pic->field, H264Picture::FIELD_NONE); 363 if (pic->long_term) { 364 pic->long_term_pic_num = pic->long_term_frame_idx; 365 } else { 366 if (pic->frame_num > frame_num) 367 pic->frame_num_wrap = pic->frame_num - max_frame_num_; 368 else 369 pic->frame_num_wrap = pic->frame_num; 370 371 pic->pic_num = pic->frame_num_wrap; 372 } 373 } 374 } 375 376 struct PicNumDescCompare { 377 bool operator()(const scoped_refptr<H264Picture>& a, 378 const scoped_refptr<H264Picture>& b) const { 379 return a->pic_num > b->pic_num; 380 } 381 }; 382 383 struct LongTermPicNumAscCompare { 384 bool operator()(const scoped_refptr<H264Picture>& a, 385 const scoped_refptr<H264Picture>& b) const { 386 return a->long_term_pic_num < b->long_term_pic_num; 387 } 388 }; 389 390 void H264Decoder::ConstructReferencePicListsP( 391 const H264SliceHeader* slice_hdr) { 392 // RefPicList0 (8.2.4.2.1) [[1] [2]], where: 393 // [1] shortterm ref pics sorted by descending pic_num, 394 // [2] longterm ref pics by ascending long_term_pic_num. 395 ref_pic_list_p0_.clear(); 396 397 // First get the short ref pics... 398 dpb_.GetShortTermRefPicsAppending(&ref_pic_list_p0_); 399 size_t num_short_refs = ref_pic_list_p0_.size(); 400 401 // and sort them to get [1]. 402 std::sort(ref_pic_list_p0_.begin(), ref_pic_list_p0_.end(), 403 PicNumDescCompare()); 404 405 // Now get long term pics and sort them by long_term_pic_num to get [2]. 406 dpb_.GetLongTermRefPicsAppending(&ref_pic_list_p0_); 407 std::sort(ref_pic_list_p0_.begin() + num_short_refs, ref_pic_list_p0_.end(), 408 LongTermPicNumAscCompare()); 409 } 410 411 struct POCAscCompare { 412 bool operator()(const scoped_refptr<H264Picture>& a, 413 const scoped_refptr<H264Picture>& b) const { 414 return a->pic_order_cnt < b->pic_order_cnt; 415 } 416 }; 417 418 struct POCDescCompare { 419 bool operator()(const scoped_refptr<H264Picture>& a, 420 const scoped_refptr<H264Picture>& b) const { 421 return a->pic_order_cnt > b->pic_order_cnt; 422 } 423 }; 424 425 void H264Decoder::ConstructReferencePicListsB( 426 const H264SliceHeader* slice_hdr) { 427 // RefPicList0 (8.2.4.2.3) [[1] [2] [3]], where: 428 // [1] shortterm ref pics with POC < curr_pic's POC sorted by descending POC, 429 // [2] shortterm ref pics with POC > curr_pic's POC by ascending POC, 430 // [3] longterm ref pics by ascending long_term_pic_num. 431 ref_pic_list_b0_.clear(); 432 ref_pic_list_b1_.clear(); 433 dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b0_); 434 size_t num_short_refs = ref_pic_list_b0_.size(); 435 436 // First sort ascending, this will put [1] in right place and finish [2]. 437 std::sort(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(), POCAscCompare()); 438 439 // Find first with POC > curr_pic's POC to get first element in [2]... 440 H264Picture::Vector::iterator iter; 441 iter = std::upper_bound(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(), 442 curr_pic_.get(), POCAscCompare()); 443 444 // and sort [1] descending, thus finishing sequence [1] [2]. 445 std::sort(ref_pic_list_b0_.begin(), iter, POCDescCompare()); 446 447 // Now add [3] and sort by ascending long_term_pic_num. 448 dpb_.GetLongTermRefPicsAppending(&ref_pic_list_b0_); 449 std::sort(ref_pic_list_b0_.begin() + num_short_refs, ref_pic_list_b0_.end(), 450 LongTermPicNumAscCompare()); 451 452 // RefPicList1 (8.2.4.2.4) [[1] [2] [3]], where: 453 // [1] shortterm ref pics with POC > curr_pic's POC sorted by ascending POC, 454 // [2] shortterm ref pics with POC < curr_pic's POC by descending POC, 455 // [3] longterm ref pics by ascending long_term_pic_num. 456 457 dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b1_); 458 num_short_refs = ref_pic_list_b1_.size(); 459 460 // First sort by descending POC. 461 std::sort(ref_pic_list_b1_.begin(), ref_pic_list_b1_.end(), POCDescCompare()); 462 463 // Find first with POC < curr_pic's POC to get first element in [2]... 464 iter = std::upper_bound(ref_pic_list_b1_.begin(), ref_pic_list_b1_.end(), 465 curr_pic_.get(), POCDescCompare()); 466 467 // and sort [1] ascending. 468 std::sort(ref_pic_list_b1_.begin(), iter, POCAscCompare()); 469 470 // Now add [3] and sort by ascending long_term_pic_num 471 dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b1_); 472 std::sort(ref_pic_list_b1_.begin() + num_short_refs, ref_pic_list_b1_.end(), 473 LongTermPicNumAscCompare()); 474 475 // If lists identical, swap first two entries in RefPicList1 (spec 8.2.4.2.3) 476 if (ref_pic_list_b1_.size() > 1 && 477 std::equal(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(), 478 ref_pic_list_b1_.begin())) 479 std::swap(ref_pic_list_b1_[0], ref_pic_list_b1_[1]); 480 } 481 482 // See 8.2.4 483 int H264Decoder::PicNumF(const scoped_refptr<H264Picture>& pic) { 484 if (!pic) 485 return -1; 486 487 if (!pic->long_term) 488 return pic->pic_num; 489 else 490 return max_pic_num_; 491 } 492 493 // See 8.2.4 494 int H264Decoder::LongTermPicNumF(const scoped_refptr<H264Picture>& pic) { 495 if (pic->ref && pic->long_term) 496 return pic->long_term_pic_num; 497 else 498 return 2 * (max_long_term_frame_idx_ + 1); 499 } 500 501 // Shift elements on the |v| starting from |from| to |to|, inclusive, 502 // one position to the right and insert pic at |from|. 503 static void ShiftRightAndInsert(H264Picture::Vector* v, 504 int from, 505 int to, 506 const scoped_refptr<H264Picture>& pic) { 507 // Security checks, do not disable in Debug mode. 508 CHECK(from <= to); 509 CHECK(to <= std::numeric_limits<int>::max() - 2); 510 // Additional checks. Debug mode ok. 511 DCHECK(v); 512 DCHECK(pic); 513 DCHECK((to + 1 == static_cast<int>(v->size())) || 514 (to + 2 == static_cast<int>(v->size()))); 515 516 v->resize(to + 2); 517 518 for (int i = to + 1; i > from; --i) 519 (*v)[i] = (*v)[i - 1]; 520 521 (*v)[from] = pic; 522 } 523 524 bool H264Decoder::ModifyReferencePicList(const H264SliceHeader* slice_hdr, 525 int list, 526 H264Picture::Vector* ref_pic_listx) { 527 bool ref_pic_list_modification_flag_lX; 528 int num_ref_idx_lX_active_minus1; 529 const H264ModificationOfPicNum* list_mod; 530 531 // This can process either ref_pic_list0 or ref_pic_list1, depending on 532 // the list argument. Set up pointers to proper list to be processed here. 533 if (list == 0) { 534 ref_pic_list_modification_flag_lX = 535 slice_hdr->ref_pic_list_modification_flag_l0; 536 num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l0_active_minus1; 537 list_mod = slice_hdr->ref_list_l0_modifications; 538 } else { 539 ref_pic_list_modification_flag_lX = 540 slice_hdr->ref_pic_list_modification_flag_l1; 541 num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l1_active_minus1; 542 list_mod = slice_hdr->ref_list_l1_modifications; 543 } 544 545 // Resize the list to the size requested in the slice header. 546 // Note that per 8.2.4.2 it's possible for num_ref_idx_lX_active_minus1 to 547 // indicate there should be more ref pics on list than we constructed. 548 // Those superfluous ones should be treated as non-reference and will be 549 // initialized to nullptr, which must be handled by clients. 550 DCHECK_GE(num_ref_idx_lX_active_minus1, 0); 551 ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1); 552 553 if (!ref_pic_list_modification_flag_lX) 554 return true; 555 556 // Spec 8.2.4.3: 557 // Reorder pictures on the list in a way specified in the stream. 558 int pic_num_lx_pred = curr_pic_->pic_num; 559 int ref_idx_lx = 0; 560 int pic_num_lx_no_wrap; 561 int pic_num_lx; 562 bool done = false; 563 scoped_refptr<H264Picture> pic; 564 for (int i = 0; i < H264SliceHeader::kRefListModSize && !done; ++i) { 565 switch (list_mod->modification_of_pic_nums_idc) { 566 case 0: 567 case 1: 568 // Modify short reference picture position. 569 if (list_mod->modification_of_pic_nums_idc == 0) { 570 // Subtract given value from predicted PicNum. 571 pic_num_lx_no_wrap = 572 pic_num_lx_pred - 573 (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1); 574 // Wrap around max_pic_num_ if it becomes < 0 as result 575 // of subtraction. 576 if (pic_num_lx_no_wrap < 0) 577 pic_num_lx_no_wrap += max_pic_num_; 578 } else { 579 // Add given value to predicted PicNum. 580 pic_num_lx_no_wrap = 581 pic_num_lx_pred + 582 (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1); 583 // Wrap around max_pic_num_ if it becomes >= max_pic_num_ as result 584 // of the addition. 585 if (pic_num_lx_no_wrap >= max_pic_num_) 586 pic_num_lx_no_wrap -= max_pic_num_; 587 } 588 589 // For use in next iteration. 590 pic_num_lx_pred = pic_num_lx_no_wrap; 591 592 if (pic_num_lx_no_wrap > curr_pic_->pic_num) 593 pic_num_lx = pic_num_lx_no_wrap - max_pic_num_; 594 else 595 pic_num_lx = pic_num_lx_no_wrap; 596 597 DCHECK_LT(num_ref_idx_lX_active_minus1 + 1, 598 H264SliceHeader::kRefListModSize); 599 pic = dpb_.GetShortRefPicByPicNum(pic_num_lx); 600 if (!pic) { 601 DVLOG(1) << "Malformed stream, no pic num " << pic_num_lx; 602 return false; 603 } 604 ShiftRightAndInsert(ref_pic_listx, ref_idx_lx, 605 num_ref_idx_lX_active_minus1, pic); 606 ref_idx_lx++; 607 608 for (int src = ref_idx_lx, dst = ref_idx_lx; 609 src <= num_ref_idx_lX_active_minus1 + 1; ++src) { 610 if (PicNumF((*ref_pic_listx)[src]) != pic_num_lx) 611 (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src]; 612 } 613 break; 614 615 case 2: 616 // Modify long term reference picture position. 617 DCHECK_LT(num_ref_idx_lX_active_minus1 + 1, 618 H264SliceHeader::kRefListModSize); 619 pic = dpb_.GetLongRefPicByLongTermPicNum(list_mod->long_term_pic_num); 620 if (!pic) { 621 DVLOG(1) << "Malformed stream, no pic num " 622 << list_mod->long_term_pic_num; 623 return false; 624 } 625 ShiftRightAndInsert(ref_pic_listx, ref_idx_lx, 626 num_ref_idx_lX_active_minus1, pic); 627 ref_idx_lx++; 628 629 for (int src = ref_idx_lx, dst = ref_idx_lx; 630 src <= num_ref_idx_lX_active_minus1 + 1; ++src) { 631 if (LongTermPicNumF((*ref_pic_listx)[src]) != 632 static_cast<int>(list_mod->long_term_pic_num)) 633 (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src]; 634 } 635 break; 636 637 case 3: 638 // End of modification list. 639 done = true; 640 break; 641 642 default: 643 // May be recoverable. 644 DVLOG(1) << "Invalid modification_of_pic_nums_idc=" 645 << list_mod->modification_of_pic_nums_idc 646 << " in position " << i; 647 break; 648 } 649 650 ++list_mod; 651 } 652 653 // Per NOTE 2 in 8.2.4.3.2, the ref_pic_listx size in the above loop is 654 // temporarily made one element longer than the required final list. 655 // Resize the list back to its required size. 656 ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1); 657 658 return true; 659 } 660 661 void H264Decoder::OutputPic(scoped_refptr<H264Picture> pic) { 662 DCHECK(!pic->outputted); 663 pic->outputted = true; 664 665 if (pic->nonexisting) { 666 DVLOG(4) << "Skipping output, non-existing frame_num: " << pic->frame_num; 667 return; 668 } 669 670 DVLOG_IF(1, pic->pic_order_cnt < last_output_poc_) 671 << "Outputting out of order, likely a broken stream: " 672 << last_output_poc_ << " -> " << pic->pic_order_cnt; 673 last_output_poc_ = pic->pic_order_cnt; 674 675 DVLOG(4) << "Posting output task for POC: " << pic->pic_order_cnt; 676 accelerator_->OutputPicture(pic); 677 } 678 679 void H264Decoder::ClearDPB() { 680 // Clear DPB contents, marking the pictures as unused first. 681 dpb_.Clear(); 682 last_output_poc_ = std::numeric_limits<int>::min(); 683 } 684 685 bool H264Decoder::OutputAllRemainingPics() { 686 // Output all pictures that are waiting to be outputted. 687 FinishPrevFrameIfPresent(); 688 H264Picture::Vector to_output; 689 dpb_.GetNotOutputtedPicsAppending(&to_output); 690 // Sort them by ascending POC to output in order. 691 std::sort(to_output.begin(), to_output.end(), POCAscCompare()); 692 693 for (auto& pic : to_output) 694 OutputPic(pic); 695 696 return true; 697 } 698 699 bool H264Decoder::Flush() { 700 DVLOG(2) << "Decoder flush"; 701 702 if (!OutputAllRemainingPics()) 703 return false; 704 705 ClearDPB(); 706 DVLOG(2) << "Decoder flush finished"; 707 return true; 708 } 709 710 bool H264Decoder::StartNewFrame(const H264SliceHeader* slice_hdr) { 711 // TODO posciak: add handling of max_num_ref_frames per spec. 712 CHECK(curr_pic_.get()); 713 DCHECK(slice_hdr); 714 715 curr_pps_id_ = slice_hdr->pic_parameter_set_id; 716 const H264PPS* pps = parser_.GetPPS(curr_pps_id_); 717 if (!pps) 718 return false; 719 720 curr_sps_id_ = pps->seq_parameter_set_id; 721 const H264SPS* sps = parser_.GetSPS(curr_sps_id_); 722 if (!sps) 723 return false; 724 725 max_frame_num_ = 1 << (sps->log2_max_frame_num_minus4 + 4); 726 int frame_num = slice_hdr->frame_num; 727 if (slice_hdr->idr_pic_flag) 728 prev_ref_frame_num_ = 0; 729 730 // 7.4.3 731 if (frame_num != prev_ref_frame_num_ && 732 frame_num != (prev_ref_frame_num_ + 1) % max_frame_num_) { 733 if (!HandleFrameNumGap(frame_num)) 734 return false; 735 } 736 737 if (!InitCurrPicture(slice_hdr)) 738 return false; 739 740 UpdatePicNums(frame_num); 741 PrepareRefPicLists(slice_hdr); 742 743 if (!accelerator_->SubmitFrameMetadata(sps, pps, dpb_, ref_pic_list_p0_, 744 ref_pic_list_b0_, ref_pic_list_b1_, 745 curr_pic_.get())) 746 return false; 747 748 return true; 749 } 750 751 bool H264Decoder::HandleMemoryManagementOps(scoped_refptr<H264Picture> pic) { 752 // 8.2.5.4 753 for (size_t i = 0; i < arraysize(pic->ref_pic_marking); ++i) { 754 // Code below does not support interlaced stream (per-field pictures). 755 H264DecRefPicMarking* ref_pic_marking = &pic->ref_pic_marking[i]; 756 scoped_refptr<H264Picture> to_mark; 757 int pic_num_x; 758 759 switch (ref_pic_marking->memory_mgmnt_control_operation) { 760 case 0: 761 // Normal end of operations' specification. 762 return true; 763 764 case 1: 765 // Mark a short term reference picture as unused so it can be removed 766 // if outputted. 767 pic_num_x = 768 pic->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + 1); 769 to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x); 770 if (to_mark) { 771 to_mark->ref = false; 772 } else { 773 DVLOG(1) << "Invalid short ref pic num to unmark"; 774 return false; 775 } 776 break; 777 778 case 2: 779 // Mark a long term reference picture as unused so it can be removed 780 // if outputted. 781 to_mark = dpb_.GetLongRefPicByLongTermPicNum( 782 ref_pic_marking->long_term_pic_num); 783 if (to_mark) { 784 to_mark->ref = false; 785 } else { 786 DVLOG(1) << "Invalid long term ref pic num to unmark"; 787 return false; 788 } 789 break; 790 791 case 3: 792 // Mark a short term reference picture as long term reference. 793 pic_num_x = 794 pic->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + 1); 795 to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x); 796 if (to_mark) { 797 DCHECK(to_mark->ref && !to_mark->long_term); 798 to_mark->long_term = true; 799 to_mark->long_term_frame_idx = ref_pic_marking->long_term_frame_idx; 800 } else { 801 DVLOG(1) << "Invalid short term ref pic num to mark as long ref"; 802 return false; 803 } 804 break; 805 806 case 4: { 807 // Unmark all reference pictures with long_term_frame_idx over new max. 808 max_long_term_frame_idx_ = 809 ref_pic_marking->max_long_term_frame_idx_plus1 - 1; 810 H264Picture::Vector long_terms; 811 dpb_.GetLongTermRefPicsAppending(&long_terms); 812 for (size_t i = 0; i < long_terms.size(); ++i) { 813 scoped_refptr<H264Picture>& long_term_pic = long_terms[i]; 814 DCHECK(long_term_pic->ref && long_term_pic->long_term); 815 // Ok to cast, max_long_term_frame_idx is much smaller than 16bit. 816 if (long_term_pic->long_term_frame_idx > 817 static_cast<int>(max_long_term_frame_idx_)) 818 long_term_pic->ref = false; 819 } 820 break; 821 } 822 823 case 5: 824 // Unmark all reference pictures. 825 dpb_.MarkAllUnusedForRef(); 826 max_long_term_frame_idx_ = -1; 827 pic->mem_mgmt_5 = true; 828 break; 829 830 case 6: { 831 // Replace long term reference pictures with current picture. 832 // First unmark if any existing with this long_term_frame_idx... 833 H264Picture::Vector long_terms; 834 dpb_.GetLongTermRefPicsAppending(&long_terms); 835 for (size_t i = 0; i < long_terms.size(); ++i) { 836 scoped_refptr<H264Picture>& long_term_pic = long_terms[i]; 837 DCHECK(long_term_pic->ref && long_term_pic->long_term); 838 // Ok to cast, long_term_frame_idx is much smaller than 16bit. 839 if (long_term_pic->long_term_frame_idx == 840 static_cast<int>(ref_pic_marking->long_term_frame_idx)) 841 long_term_pic->ref = false; 842 } 843 844 // and mark the current one instead. 845 pic->ref = true; 846 pic->long_term = true; 847 pic->long_term_frame_idx = ref_pic_marking->long_term_frame_idx; 848 break; 849 } 850 851 default: 852 // Would indicate a bug in parser. 853 NOTREACHED(); 854 } 855 } 856 857 return true; 858 } 859 860 // This method ensures that DPB does not overflow, either by removing 861 // reference pictures as specified in the stream, or using a sliding window 862 // procedure to remove the oldest one. 863 // It also performs marking and unmarking pictures as reference. 864 // See spac 8.2.5.1. 865 bool H264Decoder::ReferencePictureMarking(scoped_refptr<H264Picture> pic) { 866 // If the current picture is an IDR, all reference pictures are unmarked. 867 if (pic->idr) { 868 dpb_.MarkAllUnusedForRef(); 869 870 if (pic->long_term_reference_flag) { 871 pic->long_term = true; 872 pic->long_term_frame_idx = 0; 873 max_long_term_frame_idx_ = 0; 874 } else { 875 pic->long_term = false; 876 max_long_term_frame_idx_ = -1; 877 } 878 879 return true; 880 } 881 882 // Not an IDR. If the stream contains instructions on how to discard pictures 883 // from DPB and how to mark/unmark existing reference pictures, do so. 884 // Otherwise, fall back to default sliding window process. 885 if (pic->adaptive_ref_pic_marking_mode_flag) { 886 DCHECK(!pic->nonexisting); 887 return HandleMemoryManagementOps(pic); 888 } else { 889 return SlidingWindowPictureMarking(); 890 } 891 } 892 893 bool H264Decoder::SlidingWindowPictureMarking() { 894 const H264SPS* sps = parser_.GetSPS(curr_sps_id_); 895 if (!sps) 896 return false; 897 898 // 8.2.5.3. Ensure the DPB doesn't overflow by discarding the oldest picture. 899 int num_ref_pics = dpb_.CountRefPics(); 900 DCHECK_LE(num_ref_pics, std::max<int>(sps->max_num_ref_frames, 1)); 901 if (num_ref_pics == std::max<int>(sps->max_num_ref_frames, 1)) { 902 // Max number of reference pics reached, need to remove one of the short 903 // term ones. Find smallest frame_num_wrap short reference picture and mark 904 // it as unused. 905 scoped_refptr<H264Picture> to_unmark = 906 dpb_.GetLowestFrameNumWrapShortRefPic(); 907 if (!to_unmark) { 908 DVLOG(1) << "Couldn't find a short ref picture to unmark"; 909 return false; 910 } 911 912 to_unmark->ref = false; 913 } 914 915 return true; 916 } 917 918 bool H264Decoder::FinishPicture(scoped_refptr<H264Picture> pic) { 919 // Finish processing the picture. 920 // Start by storing previous picture data for later use. 921 if (pic->ref) { 922 ReferencePictureMarking(pic); 923 prev_ref_has_memmgmnt5_ = pic->mem_mgmt_5; 924 prev_ref_top_field_order_cnt_ = pic->top_field_order_cnt; 925 prev_ref_pic_order_cnt_msb_ = pic->pic_order_cnt_msb; 926 prev_ref_pic_order_cnt_lsb_ = pic->pic_order_cnt_lsb; 927 prev_ref_field_ = pic->field; 928 prev_ref_frame_num_ = pic->frame_num; 929 } 930 prev_frame_num_ = pic->frame_num; 931 prev_has_memmgmnt5_ = pic->mem_mgmt_5; 932 prev_frame_num_offset_ = pic->frame_num_offset; 933 934 // Remove unused (for reference or later output) pictures from DPB, marking 935 // them as such. 936 dpb_.DeleteUnused(); 937 938 DVLOG(4) << "Finishing picture frame_num: " << pic->frame_num 939 << ", entries in DPB: " << dpb_.size(); 940 941 // The ownership of pic will either be transferred to DPB - if the picture is 942 // still needed (for output and/or reference) - or we will release it 943 // immediately if we manage to output it here and won't have to store it for 944 // future reference. 945 946 // Get all pictures that haven't been outputted yet. 947 H264Picture::Vector not_outputted; 948 dpb_.GetNotOutputtedPicsAppending(¬_outputted); 949 // Include the one we've just decoded. 950 not_outputted.push_back(pic); 951 952 // Sort in output order. 953 std::sort(not_outputted.begin(), not_outputted.end(), POCAscCompare()); 954 955 // Try to output as many pictures as we can. A picture can be output, 956 // if the number of decoded and not yet outputted pictures that would remain 957 // in DPB afterwards would at least be equal to max_num_reorder_frames. 958 // If the outputted picture is not a reference picture, it doesn't have 959 // to remain in the DPB and can be removed. 960 H264Picture::Vector::iterator output_candidate = not_outputted.begin(); 961 size_t num_remaining = not_outputted.size(); 962 while (num_remaining > max_num_reorder_frames_ || 963 // If the condition below is used, this is an invalid stream. We should 964 // not be forced to output beyond max_num_reorder_frames in order to 965 // make room in DPB to store the current picture (if we need to do so). 966 // However, if this happens, ignore max_num_reorder_frames and try 967 // to output more. This may cause out-of-order output, but is not 968 // fatal, and better than failing instead. 969 ((dpb_.IsFull() && (!pic->outputted || pic->ref)) && num_remaining)) { 970 DVLOG_IF(1, num_remaining <= max_num_reorder_frames_) 971 << "Invalid stream: max_num_reorder_frames not preserved"; 972 973 OutputPic(*output_candidate); 974 975 if (!(*output_candidate)->ref) { 976 // Current picture hasn't been inserted into DPB yet, so don't remove it 977 // if we managed to output it immediately. 978 int outputted_poc = (*output_candidate)->pic_order_cnt; 979 if (outputted_poc != pic->pic_order_cnt) 980 dpb_.DeleteByPOC(outputted_poc); 981 } 982 983 ++output_candidate; 984 --num_remaining; 985 } 986 987 // If we haven't managed to output the picture that we just decoded, or if 988 // it's a reference picture, we have to store it in DPB. 989 if (!pic->outputted || pic->ref) { 990 if (dpb_.IsFull()) { 991 // If we haven't managed to output anything to free up space in DPB 992 // to store this picture, it's an error in the stream. 993 DVLOG(1) << "Could not free up space in DPB!"; 994 return false; 995 } 996 997 dpb_.StorePic(pic); 998 } 999 1000 return true; 1001 } 1002 1003 static int LevelToMaxDpbMbs(int level) { 1004 // See table A-1 in spec. 1005 switch (level) { 1006 case 10: 1007 return 396; 1008 case 11: 1009 return 900; 1010 case 12: // fallthrough 1011 case 13: // fallthrough 1012 case 20: 1013 return 2376; 1014 case 21: 1015 return 4752; 1016 case 22: // fallthrough 1017 case 30: 1018 return 8100; 1019 case 31: 1020 return 18000; 1021 case 32: 1022 return 20480; 1023 case 40: // fallthrough 1024 case 41: 1025 return 32768; 1026 case 42: 1027 return 34816; 1028 case 50: 1029 return 110400; 1030 case 51: // fallthrough 1031 case 52: 1032 return 184320; 1033 default: 1034 DVLOG(1) << "Invalid codec level (" << level << ")"; 1035 return 0; 1036 } 1037 } 1038 1039 bool H264Decoder::UpdateMaxNumReorderFrames(const H264SPS* sps) { 1040 if (sps->vui_parameters_present_flag && sps->bitstream_restriction_flag) { 1041 max_num_reorder_frames_ = 1042 base::checked_cast<size_t>(sps->max_num_reorder_frames); 1043 if (max_num_reorder_frames_ > dpb_.max_num_pics()) { 1044 DVLOG(1) 1045 << "max_num_reorder_frames present, but larger than MaxDpbFrames (" 1046 << max_num_reorder_frames_ << " > " << dpb_.max_num_pics() << ")"; 1047 max_num_reorder_frames_ = 0; 1048 return false; 1049 } 1050 return true; 1051 } 1052 1053 // max_num_reorder_frames not present, infer from profile/constraints 1054 // (see VUI semantics in spec). 1055 if (sps->constraint_set3_flag) { 1056 switch (sps->profile_idc) { 1057 case 44: 1058 case 86: 1059 case 100: 1060 case 110: 1061 case 122: 1062 case 244: 1063 max_num_reorder_frames_ = 0; 1064 break; 1065 default: 1066 max_num_reorder_frames_ = dpb_.max_num_pics(); 1067 break; 1068 } 1069 } else { 1070 max_num_reorder_frames_ = dpb_.max_num_pics(); 1071 } 1072 1073 return true; 1074 } 1075 1076 bool H264Decoder::ProcessSPS(int sps_id, bool* need_new_buffers) { 1077 DVLOG(4) << "Processing SPS id:" << sps_id; 1078 1079 const H264SPS* sps = parser_.GetSPS(sps_id); 1080 if (!sps) 1081 return false; 1082 1083 *need_new_buffers = false; 1084 1085 if (sps->frame_mbs_only_flag == 0) { 1086 DVLOG(1) << "frame_mbs_only_flag != 1 not supported"; 1087 return false; 1088 } 1089 1090 Size new_pic_size = sps->GetCodedSize().value_or(Size()); 1091 if (new_pic_size.IsEmpty()) { 1092 DVLOG(1) << "Invalid picture size"; 1093 return false; 1094 } 1095 1096 int width_mb = new_pic_size.width() / 16; 1097 int height_mb = new_pic_size.height() / 16; 1098 1099 // Verify that the values are not too large before multiplying. 1100 if (std::numeric_limits<int>::max() / width_mb < height_mb) { 1101 DVLOG(1) << "Picture size is too big: " << new_pic_size.ToString(); 1102 return false; 1103 } 1104 1105 int level = sps->level_idc; 1106 int max_dpb_mbs = LevelToMaxDpbMbs(level); 1107 if (max_dpb_mbs == 0) 1108 return false; 1109 1110 size_t max_dpb_size = std::min(max_dpb_mbs / (width_mb * height_mb), 1111 static_cast<int>(H264DPB::kDPBMaxSize)); 1112 if (max_dpb_size == 0) { 1113 DVLOG(1) << "Invalid DPB Size"; 1114 return false; 1115 } 1116 1117 if ((pic_size_ != new_pic_size) || (dpb_.max_num_pics() != max_dpb_size)) { 1118 if (!Flush()) 1119 return false; 1120 DVLOG(1) << "Codec level: " << level << ", DPB size: " << max_dpb_size 1121 << ", Picture size: " << new_pic_size.ToString(); 1122 *need_new_buffers = true; 1123 pic_size_ = new_pic_size; 1124 dpb_.set_max_num_pics(max_dpb_size); 1125 } 1126 1127 if (!UpdateMaxNumReorderFrames(sps)) 1128 return false; 1129 DVLOG(1) << "max_num_reorder_frames: " << max_num_reorder_frames_; 1130 1131 return true; 1132 } 1133 1134 bool H264Decoder::FinishPrevFrameIfPresent() { 1135 // If we already have a frame waiting to be decoded, decode it and finish. 1136 if (curr_pic_) { 1137 if (!DecodePicture()) 1138 return false; 1139 1140 scoped_refptr<H264Picture> pic = curr_pic_; 1141 curr_pic_ = nullptr; 1142 return FinishPicture(pic); 1143 } 1144 1145 return true; 1146 } 1147 1148 bool H264Decoder::HandleFrameNumGap(int frame_num) { 1149 const H264SPS* sps = parser_.GetSPS(curr_sps_id_); 1150 if (!sps) 1151 return false; 1152 1153 if (!sps->gaps_in_frame_num_value_allowed_flag) { 1154 DVLOG(1) << "Invalid frame_num: " << frame_num; 1155 return false; 1156 } 1157 1158 DVLOG(2) << "Handling frame_num gap: " << prev_ref_frame_num_ << "->" 1159 << frame_num; 1160 1161 // 7.4.3/7-23 1162 int unused_short_term_frame_num = (prev_ref_frame_num_ + 1) % max_frame_num_; 1163 while (unused_short_term_frame_num != frame_num) { 1164 scoped_refptr<H264Picture> pic = new H264Picture(); 1165 if (!InitNonexistingPicture(pic, unused_short_term_frame_num)) 1166 return false; 1167 1168 UpdatePicNums(unused_short_term_frame_num); 1169 1170 if (!FinishPicture(pic)) 1171 return false; 1172 1173 unused_short_term_frame_num++; 1174 unused_short_term_frame_num %= max_frame_num_; 1175 } 1176 1177 return true; 1178 } 1179 1180 bool H264Decoder::IsNewPrimaryCodedPicture( 1181 const H264SliceHeader* slice_hdr) const { 1182 if (!curr_pic_) 1183 return true; 1184 1185 // 7.4.1.2.4, assumes non-interlaced. 1186 if (slice_hdr->frame_num != curr_pic_->frame_num || 1187 slice_hdr->pic_parameter_set_id != curr_pps_id_ || 1188 slice_hdr->nal_ref_idc != curr_pic_->nal_ref_idc || 1189 slice_hdr->idr_pic_flag != curr_pic_->idr || 1190 (slice_hdr->idr_pic_flag && 1191 (slice_hdr->idr_pic_id != curr_pic_->idr_pic_id || 1192 // If we have two consecutive IDR slices, and the second one has 1193 // first_mb_in_slice == 0, treat it as a new picture. 1194 // Per spec, idr_pic_id should not be equal in this case (and we should 1195 // have hit the condition above instead, see spec 7.4.3 on idr_pic_id), 1196 // but some encoders neglect changing idr_pic_id for two consecutive 1197 // IDRs. Work around this by checking if the next slice contains the 1198 // zeroth macroblock, i.e. data that belongs to the next picture. 1199 slice_hdr->first_mb_in_slice == 0))) 1200 return true; 1201 1202 const H264SPS* sps = parser_.GetSPS(curr_sps_id_); 1203 if (!sps) 1204 return false; 1205 1206 if (sps->pic_order_cnt_type == curr_pic_->pic_order_cnt_type) { 1207 if (curr_pic_->pic_order_cnt_type == 0) { 1208 if (slice_hdr->pic_order_cnt_lsb != curr_pic_->pic_order_cnt_lsb || 1209 slice_hdr->delta_pic_order_cnt_bottom != 1210 curr_pic_->delta_pic_order_cnt_bottom) 1211 return true; 1212 } else if (curr_pic_->pic_order_cnt_type == 1) { 1213 if (slice_hdr->delta_pic_order_cnt0 != curr_pic_->delta_pic_order_cnt0 || 1214 slice_hdr->delta_pic_order_cnt1 != curr_pic_->delta_pic_order_cnt1) 1215 return true; 1216 } 1217 } 1218 1219 return false; 1220 } 1221 1222 bool H264Decoder::PreprocessCurrentSlice() { 1223 const H264SliceHeader* slice_hdr = curr_slice_hdr_.get(); 1224 DCHECK(slice_hdr); 1225 1226 if (IsNewPrimaryCodedPicture(slice_hdr)) { 1227 // New picture, so first finish the previous one before processing it. 1228 if (!FinishPrevFrameIfPresent()) 1229 return false; 1230 1231 DCHECK(!curr_pic_); 1232 1233 if (slice_hdr->first_mb_in_slice != 0) { 1234 DVLOG(1) << "ASO/invalid stream, first_mb_in_slice: " 1235 << slice_hdr->first_mb_in_slice; 1236 return false; 1237 } 1238 1239 // If the new picture is an IDR, flush DPB. 1240 if (slice_hdr->idr_pic_flag) { 1241 // Output all remaining pictures, unless we are explicitly instructed 1242 // not to do so. 1243 if (!slice_hdr->no_output_of_prior_pics_flag) { 1244 if (!Flush()) 1245 return false; 1246 } 1247 dpb_.Clear(); 1248 last_output_poc_ = std::numeric_limits<int>::min(); 1249 } 1250 } 1251 1252 return true; 1253 } 1254 1255 bool H264Decoder::ProcessCurrentSlice() { 1256 DCHECK(curr_pic_); 1257 1258 const H264SliceHeader* slice_hdr = curr_slice_hdr_.get(); 1259 DCHECK(slice_hdr); 1260 1261 if (slice_hdr->field_pic_flag == 0) 1262 max_pic_num_ = max_frame_num_; 1263 else 1264 max_pic_num_ = 2 * max_frame_num_; 1265 1266 H264Picture::Vector ref_pic_list0, ref_pic_list1; 1267 if (!ModifyReferencePicLists(slice_hdr, &ref_pic_list0, &ref_pic_list1)) 1268 return false; 1269 1270 const H264PPS* pps = parser_.GetPPS(curr_pps_id_); 1271 if (!pps) 1272 return false; 1273 1274 if (!accelerator_->SubmitSlice(pps, slice_hdr, ref_pic_list0, ref_pic_list1, 1275 curr_pic_.get(), slice_hdr->nalu_data, 1276 slice_hdr->nalu_size)) 1277 return false; 1278 1279 return true; 1280 } 1281 1282 #define SET_ERROR_AND_RETURN() \ 1283 do { \ 1284 DVLOG(1) << "Error during decode"; \ 1285 state_ = kError; \ 1286 return H264Decoder::kDecodeError; \ 1287 } while (0) 1288 1289 void H264Decoder::SetStream(const uint8_t* ptr, size_t size) { 1290 DCHECK(ptr); 1291 DCHECK(size); 1292 1293 DVLOG(4) << "New input stream at: " << (void*)ptr << " size: " << size; 1294 parser_.SetStream(ptr, size); 1295 } 1296 1297 H264Decoder::DecodeResult H264Decoder::Decode() { 1298 if (state_ == kError) { 1299 DVLOG(1) << "Decoder in error state"; 1300 return kDecodeError; 1301 } 1302 1303 while (1) { 1304 H264Parser::Result par_res; 1305 1306 if (!curr_nalu_) { 1307 curr_nalu_.reset(new H264NALU()); 1308 par_res = parser_.AdvanceToNextNALU(curr_nalu_.get()); 1309 if (par_res == H264Parser::kEOStream) 1310 return kRanOutOfStreamData; 1311 else if (par_res != H264Parser::kOk) 1312 SET_ERROR_AND_RETURN(); 1313 1314 DVLOG(4) << "New NALU: " << static_cast<int>(curr_nalu_->nal_unit_type); 1315 } 1316 1317 switch (curr_nalu_->nal_unit_type) { 1318 case H264NALU::kNonIDRSlice: 1319 // We can't resume from a non-IDR slice. 1320 if (state_ != kDecoding) 1321 break; 1322 1323 // else fallthrough 1324 case H264NALU::kIDRSlice: { 1325 // TODO(posciak): the IDR may require an SPS that we don't have 1326 // available. For now we'd fail if that happens, but ideally we'd like 1327 // to keep going until the next SPS in the stream. 1328 if (state_ == kNeedStreamMetadata) { 1329 // We need an SPS, skip this IDR and keep looking. 1330 break; 1331 } 1332 1333 // If after reset, we should be able to recover from an IDR. 1334 state_ = kDecoding; 1335 1336 if (!curr_slice_hdr_) { 1337 curr_slice_hdr_.reset(new H264SliceHeader()); 1338 par_res = 1339 parser_.ParseSliceHeader(*curr_nalu_, curr_slice_hdr_.get()); 1340 if (par_res != H264Parser::kOk) 1341 SET_ERROR_AND_RETURN(); 1342 1343 if (!PreprocessCurrentSlice()) 1344 SET_ERROR_AND_RETURN(); 1345 } 1346 1347 if (!curr_pic_) { 1348 // New picture/finished previous one, try to start a new one 1349 // or tell the client we need more surfaces. 1350 curr_pic_ = accelerator_->CreateH264Picture(); 1351 if (!curr_pic_) 1352 return kRanOutOfSurfaces; 1353 1354 if (!StartNewFrame(curr_slice_hdr_.get())) 1355 SET_ERROR_AND_RETURN(); 1356 } 1357 1358 if (!ProcessCurrentSlice()) 1359 SET_ERROR_AND_RETURN(); 1360 1361 curr_slice_hdr_.reset(); 1362 break; 1363 } 1364 1365 case H264NALU::kSPS: { 1366 int sps_id; 1367 1368 if (!FinishPrevFrameIfPresent()) 1369 SET_ERROR_AND_RETURN(); 1370 1371 par_res = parser_.ParseSPS(&sps_id); 1372 if (par_res != H264Parser::kOk) 1373 SET_ERROR_AND_RETURN(); 1374 1375 bool need_new_buffers = false; 1376 if (!ProcessSPS(sps_id, &need_new_buffers)) 1377 SET_ERROR_AND_RETURN(); 1378 1379 if (state_ == kNeedStreamMetadata) 1380 state_ = kAfterReset; 1381 1382 if (need_new_buffers) { 1383 curr_pic_ = nullptr; 1384 curr_nalu_ = nullptr; 1385 ref_pic_list_p0_.clear(); 1386 ref_pic_list_b0_.clear(); 1387 ref_pic_list_b1_.clear(); 1388 1389 return kAllocateNewSurfaces; 1390 } 1391 break; 1392 } 1393 1394 case H264NALU::kPPS: { 1395 int pps_id; 1396 1397 if (!FinishPrevFrameIfPresent()) 1398 SET_ERROR_AND_RETURN(); 1399 1400 par_res = parser_.ParsePPS(&pps_id); 1401 if (par_res != H264Parser::kOk) 1402 SET_ERROR_AND_RETURN(); 1403 1404 break; 1405 } 1406 1407 case H264NALU::kAUD: 1408 case H264NALU::kEOSeq: 1409 case H264NALU::kEOStream: 1410 if (state_ != kDecoding) 1411 break; 1412 1413 if (!FinishPrevFrameIfPresent()) 1414 SET_ERROR_AND_RETURN(); 1415 1416 break; 1417 1418 default: 1419 DVLOG(4) << "Skipping NALU type: " << curr_nalu_->nal_unit_type; 1420 break; 1421 } 1422 1423 DVLOG(4) << "NALU done"; 1424 curr_nalu_.reset(); 1425 } 1426 } 1427 1428 Size H264Decoder::GetPicSize() const { 1429 return pic_size_; 1430 } 1431 1432 size_t H264Decoder::GetRequiredNumOfPictures() const { 1433 return dpb_.max_num_pics() + kPicsInPipeline; 1434 } 1435 1436 } // namespace media 1437