1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include <algorithm> 6 #include <limits> 7 8 #include "base/bind.h" 9 #include "base/bind_helpers.h" 10 #include "base/stl_util.h" 11 #include "content/common/gpu/media/vaapi_h264_decoder.h" 12 13 namespace content { 14 15 // Decode surface, used for decoding and reference. input_id comes from client 16 // and is associated with the surface that was produced as the result 17 // of decoding a bitstream buffer with that id. 18 class VaapiH264Decoder::DecodeSurface { 19 public: 20 DecodeSurface(int poc, 21 int32 input_id, 22 const scoped_refptr<VASurface>& va_surface); 23 DecodeSurface(int poc, const scoped_refptr<DecodeSurface>& dec_surface); 24 ~DecodeSurface(); 25 26 int poc() { 27 return poc_; 28 } 29 30 scoped_refptr<VASurface> va_surface() { 31 return va_surface_; 32 } 33 34 int32 input_id() { 35 return input_id_; 36 } 37 38 private: 39 int poc_; 40 int32 input_id_; 41 scoped_refptr<VASurface> va_surface_; 42 }; 43 44 VaapiH264Decoder::DecodeSurface::DecodeSurface( 45 int poc, 46 int32 input_id, 47 const scoped_refptr<VASurface>& va_surface) 48 : poc_(poc), 49 input_id_(input_id), 50 va_surface_(va_surface) { 51 DCHECK(va_surface_.get()); 52 } 53 54 VaapiH264Decoder::DecodeSurface::~DecodeSurface() { 55 } 56 57 VaapiH264Decoder::VaapiH264Decoder( 58 VaapiWrapper* vaapi_wrapper, 59 const OutputPicCB& output_pic_cb, 60 const ReportErrorToUmaCB& report_error_to_uma_cb) 61 : max_pic_order_cnt_lsb_(0), 62 max_frame_num_(0), 63 max_pic_num_(0), 64 max_long_term_frame_idx_(0), 65 curr_sps_id_(-1), 66 curr_pps_id_(-1), 67 vaapi_wrapper_(vaapi_wrapper), 68 output_pic_cb_(output_pic_cb), 69 report_error_to_uma_cb_(report_error_to_uma_cb) { 70 Reset(); 71 state_ = kNeedStreamMetadata; 72 } 73 74 VaapiH264Decoder::~VaapiH264Decoder() { 75 } 76 77 void VaapiH264Decoder::Reset() { 78 curr_pic_.reset(); 79 80 curr_input_id_ = -1; 81 frame_num_ = 0; 82 prev_frame_num_ = -1; 83 prev_frame_num_offset_ = -1; 84 85 prev_ref_has_memmgmnt5_ = false; 86 prev_ref_top_field_order_cnt_ = -1; 87 prev_ref_pic_order_cnt_msb_ = -1; 88 prev_ref_pic_order_cnt_lsb_ = -1; 89 prev_ref_field_ = H264Picture::FIELD_NONE; 90 91 vaapi_wrapper_->DestroyPendingBuffers(); 92 93 ref_pic_list0_.clear(); 94 ref_pic_list1_.clear(); 95 96 for (DecSurfacesInUse::iterator it = decode_surfaces_in_use_.begin(); 97 it != decode_surfaces_in_use_.end(); ) { 98 int poc = it->second->poc(); 99 // Must be incremented before UnassignSurfaceFromPoC as this call 100 // invalidates |it|. 101 ++it; 102 UnassignSurfaceFromPoC(poc); 103 } 104 DCHECK(decode_surfaces_in_use_.empty()); 105 106 dpb_.Clear(); 107 parser_.Reset(); 108 last_output_poc_ = 0; 109 110 // If we are in kDecoding, we can resume without processing an SPS. 111 if (state_ == kDecoding) 112 state_ = kAfterReset; 113 } 114 115 void VaapiH264Decoder::ReuseSurface( 116 const scoped_refptr<VASurface>& va_surface) { 117 available_va_surfaces_.push_back(va_surface); 118 } 119 120 // Fill |va_pic| with default/neutral values. 121 static void InitVAPicture(VAPictureH264* va_pic) { 122 memset(va_pic, 0, sizeof(*va_pic)); 123 va_pic->picture_id = VA_INVALID_ID; 124 va_pic->flags = VA_PICTURE_H264_INVALID; 125 } 126 127 void VaapiH264Decoder::FillVAPicture(VAPictureH264 *va_pic, H264Picture* pic) { 128 DCHECK(pic); 129 130 DecodeSurface* dec_surface = DecodeSurfaceByPoC(pic->pic_order_cnt); 131 if (!dec_surface) { 132 // Cannot provide a ref picture, will corrupt output, but may be able 133 // to recover. 134 InitVAPicture(va_pic); 135 return; 136 } 137 138 va_pic->picture_id = dec_surface->va_surface()->id(); 139 va_pic->frame_idx = pic->frame_num; 140 va_pic->flags = 0; 141 142 switch (pic->field) { 143 case H264Picture::FIELD_NONE: 144 break; 145 case H264Picture::FIELD_TOP: 146 va_pic->flags |= VA_PICTURE_H264_TOP_FIELD; 147 break; 148 case H264Picture::FIELD_BOTTOM: 149 va_pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD; 150 break; 151 } 152 153 if (pic->ref) { 154 va_pic->flags |= pic->long_term ? VA_PICTURE_H264_LONG_TERM_REFERENCE 155 : VA_PICTURE_H264_SHORT_TERM_REFERENCE; 156 } 157 158 va_pic->TopFieldOrderCnt = pic->top_field_order_cnt; 159 va_pic->BottomFieldOrderCnt = pic->bottom_field_order_cnt; 160 } 161 162 int VaapiH264Decoder::FillVARefFramesFromDPB(VAPictureH264 *va_pics, 163 int num_pics) { 164 H264DPB::Pictures::reverse_iterator rit; 165 int i; 166 167 // Return reference frames in reverse order of insertion. 168 // Libva does not document this, but other implementations (e.g. mplayer) 169 // do it this way as well. 170 for (rit = dpb_.rbegin(), i = 0; rit != dpb_.rend() && i < num_pics; ++rit) { 171 if ((*rit)->ref) 172 FillVAPicture(&va_pics[i++], *rit); 173 } 174 175 return i; 176 } 177 178 VaapiH264Decoder::DecodeSurface* VaapiH264Decoder::DecodeSurfaceByPoC(int poc) { 179 DecSurfacesInUse::iterator iter = decode_surfaces_in_use_.find(poc); 180 if (iter == decode_surfaces_in_use_.end()) { 181 DVLOG(1) << "Could not find surface assigned to POC: " << poc; 182 return NULL; 183 } 184 185 return iter->second.get(); 186 } 187 188 bool VaapiH264Decoder::AssignSurfaceToPoC(int32 input_id, int poc) { 189 if (available_va_surfaces_.empty()) { 190 DVLOG(1) << "No VA Surfaces available"; 191 return false; 192 } 193 194 linked_ptr<DecodeSurface> dec_surface(new DecodeSurface( 195 poc, input_id, available_va_surfaces_.back())); 196 available_va_surfaces_.pop_back(); 197 198 DVLOG(4) << "POC " << poc 199 << " will use surface " << dec_surface->va_surface()->id(); 200 201 bool inserted = decode_surfaces_in_use_.insert( 202 std::make_pair(poc, dec_surface)).second; 203 DCHECK(inserted); 204 205 return true; 206 } 207 208 void VaapiH264Decoder::UnassignSurfaceFromPoC(int poc) { 209 DecSurfacesInUse::iterator it = decode_surfaces_in_use_.find(poc); 210 if (it == decode_surfaces_in_use_.end()) { 211 DVLOG(1) << "Asked to unassign an unassigned POC " << poc; 212 return; 213 } 214 215 DVLOG(4) << "POC " << poc << " no longer using VA surface " 216 << it->second->va_surface()->id(); 217 218 decode_surfaces_in_use_.erase(it); 219 } 220 221 bool VaapiH264Decoder::SendPPS() { 222 const H264PPS* pps = parser_.GetPPS(curr_pps_id_); 223 DCHECK(pps); 224 225 const H264SPS* sps = parser_.GetSPS(pps->seq_parameter_set_id); 226 DCHECK(sps); 227 228 DCHECK(curr_pic_.get()); 229 230 VAPictureParameterBufferH264 pic_param; 231 memset(&pic_param, 0, sizeof(VAPictureParameterBufferH264)); 232 233 #define FROM_SPS_TO_PP(a) pic_param.a = sps->a; 234 #define FROM_SPS_TO_PP2(a, b) pic_param.b = sps->a; 235 FROM_SPS_TO_PP2(pic_width_in_mbs_minus1, picture_width_in_mbs_minus1); 236 // This assumes non-interlaced video 237 FROM_SPS_TO_PP2(pic_height_in_map_units_minus1, 238 picture_height_in_mbs_minus1); 239 FROM_SPS_TO_PP(bit_depth_luma_minus8); 240 FROM_SPS_TO_PP(bit_depth_chroma_minus8); 241 #undef FROM_SPS_TO_PP 242 #undef FROM_SPS_TO_PP2 243 244 #define FROM_SPS_TO_PP_SF(a) pic_param.seq_fields.bits.a = sps->a; 245 #define FROM_SPS_TO_PP_SF2(a, b) pic_param.seq_fields.bits.b = sps->a; 246 FROM_SPS_TO_PP_SF(chroma_format_idc); 247 FROM_SPS_TO_PP_SF2(separate_colour_plane_flag, 248 residual_colour_transform_flag); 249 FROM_SPS_TO_PP_SF(gaps_in_frame_num_value_allowed_flag); 250 FROM_SPS_TO_PP_SF(frame_mbs_only_flag); 251 FROM_SPS_TO_PP_SF(mb_adaptive_frame_field_flag); 252 FROM_SPS_TO_PP_SF(direct_8x8_inference_flag); 253 pic_param.seq_fields.bits.MinLumaBiPredSize8x8 = (sps->level_idc >= 31); 254 FROM_SPS_TO_PP_SF(log2_max_frame_num_minus4); 255 FROM_SPS_TO_PP_SF(pic_order_cnt_type); 256 FROM_SPS_TO_PP_SF(log2_max_pic_order_cnt_lsb_minus4); 257 FROM_SPS_TO_PP_SF(delta_pic_order_always_zero_flag); 258 #undef FROM_SPS_TO_PP_SF 259 #undef FROM_SPS_TO_PP_SF2 260 261 #define FROM_PPS_TO_PP(a) pic_param.a = pps->a; 262 FROM_PPS_TO_PP(num_slice_groups_minus1); 263 pic_param.slice_group_map_type = 0; 264 pic_param.slice_group_change_rate_minus1 = 0; 265 FROM_PPS_TO_PP(pic_init_qp_minus26); 266 FROM_PPS_TO_PP(pic_init_qs_minus26); 267 FROM_PPS_TO_PP(chroma_qp_index_offset); 268 FROM_PPS_TO_PP(second_chroma_qp_index_offset); 269 #undef FROM_PPS_TO_PP 270 271 #define FROM_PPS_TO_PP_PF(a) pic_param.pic_fields.bits.a = pps->a; 272 #define FROM_PPS_TO_PP_PF2(a, b) pic_param.pic_fields.bits.b = pps->a; 273 FROM_PPS_TO_PP_PF(entropy_coding_mode_flag); 274 FROM_PPS_TO_PP_PF(weighted_pred_flag); 275 FROM_PPS_TO_PP_PF(weighted_bipred_idc); 276 FROM_PPS_TO_PP_PF(transform_8x8_mode_flag); 277 278 pic_param.pic_fields.bits.field_pic_flag = 0; 279 FROM_PPS_TO_PP_PF(constrained_intra_pred_flag); 280 FROM_PPS_TO_PP_PF2(bottom_field_pic_order_in_frame_present_flag, 281 pic_order_present_flag); 282 FROM_PPS_TO_PP_PF(deblocking_filter_control_present_flag); 283 FROM_PPS_TO_PP_PF(redundant_pic_cnt_present_flag); 284 pic_param.pic_fields.bits.reference_pic_flag = curr_pic_->ref; 285 #undef FROM_PPS_TO_PP_PF 286 #undef FROM_PPS_TO_PP_PF2 287 288 pic_param.frame_num = curr_pic_->frame_num; 289 290 InitVAPicture(&pic_param.CurrPic); 291 FillVAPicture(&pic_param.CurrPic, curr_pic_.get()); 292 293 // Init reference pictures' array. 294 for (int i = 0; i < 16; ++i) 295 InitVAPicture(&pic_param.ReferenceFrames[i]); 296 297 // And fill it with picture info from DPB. 298 FillVARefFramesFromDPB(pic_param.ReferenceFrames, 299 arraysize(pic_param.ReferenceFrames)); 300 301 pic_param.num_ref_frames = sps->max_num_ref_frames; 302 303 return vaapi_wrapper_->SubmitBuffer(VAPictureParameterBufferType, 304 sizeof(VAPictureParameterBufferH264), 305 &pic_param); 306 } 307 308 bool VaapiH264Decoder::SendIQMatrix() { 309 const H264PPS* pps = parser_.GetPPS(curr_pps_id_); 310 DCHECK(pps); 311 312 VAIQMatrixBufferH264 iq_matrix_buf; 313 memset(&iq_matrix_buf, 0, sizeof(VAIQMatrixBufferH264)); 314 315 if (pps->pic_scaling_matrix_present_flag) { 316 for (int i = 0; i < 6; ++i) { 317 for (int j = 0; j < 16; ++j) 318 iq_matrix_buf.ScalingList4x4[i][j] = pps->scaling_list4x4[i][j]; 319 } 320 321 for (int i = 0; i < 2; ++i) { 322 for (int j = 0; j < 64; ++j) 323 iq_matrix_buf.ScalingList8x8[i][j] = pps->scaling_list8x8[i][j]; 324 } 325 } else { 326 const H264SPS* sps = parser_.GetSPS(pps->seq_parameter_set_id); 327 DCHECK(sps); 328 for (int i = 0; i < 6; ++i) { 329 for (int j = 0; j < 16; ++j) 330 iq_matrix_buf.ScalingList4x4[i][j] = sps->scaling_list4x4[i][j]; 331 } 332 333 for (int i = 0; i < 2; ++i) { 334 for (int j = 0; j < 64; ++j) 335 iq_matrix_buf.ScalingList8x8[i][j] = sps->scaling_list8x8[i][j]; 336 } 337 } 338 339 return vaapi_wrapper_->SubmitBuffer(VAIQMatrixBufferType, 340 sizeof(VAIQMatrixBufferH264), 341 &iq_matrix_buf); 342 } 343 344 bool VaapiH264Decoder::SendVASliceParam(H264SliceHeader* slice_hdr) { 345 const H264PPS* pps = parser_.GetPPS(slice_hdr->pic_parameter_set_id); 346 DCHECK(pps); 347 348 const H264SPS* sps = parser_.GetSPS(pps->seq_parameter_set_id); 349 DCHECK(sps); 350 351 VASliceParameterBufferH264 slice_param; 352 memset(&slice_param, 0, sizeof(VASliceParameterBufferH264)); 353 354 slice_param.slice_data_size = slice_hdr->nalu_size; 355 slice_param.slice_data_offset = 0; 356 slice_param.slice_data_flag = VA_SLICE_DATA_FLAG_ALL; 357 slice_param.slice_data_bit_offset = slice_hdr->header_bit_size; 358 359 #define SHDRToSP(a) slice_param.a = slice_hdr->a; 360 SHDRToSP(first_mb_in_slice); 361 slice_param.slice_type = slice_hdr->slice_type % 5; 362 SHDRToSP(direct_spatial_mv_pred_flag); 363 364 // TODO posciak: make sure parser sets those even when override flags 365 // in slice header is off. 366 SHDRToSP(num_ref_idx_l0_active_minus1); 367 SHDRToSP(num_ref_idx_l1_active_minus1); 368 SHDRToSP(cabac_init_idc); 369 SHDRToSP(slice_qp_delta); 370 SHDRToSP(disable_deblocking_filter_idc); 371 SHDRToSP(slice_alpha_c0_offset_div2); 372 SHDRToSP(slice_beta_offset_div2); 373 374 if (((slice_hdr->IsPSlice() || slice_hdr->IsSPSlice()) && 375 pps->weighted_pred_flag) || 376 (slice_hdr->IsBSlice() && pps->weighted_bipred_idc == 1)) { 377 SHDRToSP(luma_log2_weight_denom); 378 SHDRToSP(chroma_log2_weight_denom); 379 380 SHDRToSP(luma_weight_l0_flag); 381 SHDRToSP(luma_weight_l1_flag); 382 383 SHDRToSP(chroma_weight_l0_flag); 384 SHDRToSP(chroma_weight_l1_flag); 385 386 for (int i = 0; i <= slice_param.num_ref_idx_l0_active_minus1; ++i) { 387 slice_param.luma_weight_l0[i] = 388 slice_hdr->pred_weight_table_l0.luma_weight[i]; 389 slice_param.luma_offset_l0[i] = 390 slice_hdr->pred_weight_table_l0.luma_offset[i]; 391 392 for (int j = 0; j < 2; ++j) { 393 slice_param.chroma_weight_l0[i][j] = 394 slice_hdr->pred_weight_table_l0.chroma_weight[i][j]; 395 slice_param.chroma_offset_l0[i][j] = 396 slice_hdr->pred_weight_table_l0.chroma_offset[i][j]; 397 } 398 } 399 400 if (slice_hdr->IsBSlice()) { 401 for (int i = 0; i <= slice_param.num_ref_idx_l1_active_minus1; ++i) { 402 slice_param.luma_weight_l1[i] = 403 slice_hdr->pred_weight_table_l1.luma_weight[i]; 404 slice_param.luma_offset_l1[i] = 405 slice_hdr->pred_weight_table_l1.luma_offset[i]; 406 407 for (int j = 0; j < 2; ++j) { 408 slice_param.chroma_weight_l1[i][j] = 409 slice_hdr->pred_weight_table_l1.chroma_weight[i][j]; 410 slice_param.chroma_offset_l1[i][j] = 411 slice_hdr->pred_weight_table_l1.chroma_offset[i][j]; 412 } 413 } 414 } 415 } 416 417 for (int i = 0; i < 32; ++i) { 418 InitVAPicture(&slice_param.RefPicList0[i]); 419 InitVAPicture(&slice_param.RefPicList1[i]); 420 } 421 422 int i; 423 H264Picture::PtrVector::iterator it; 424 for (it = ref_pic_list0_.begin(), i = 0; it != ref_pic_list0_.end() && *it; 425 ++it, ++i) 426 FillVAPicture(&slice_param.RefPicList0[i], *it); 427 for (it = ref_pic_list1_.begin(), i = 0; it != ref_pic_list1_.end() && *it; 428 ++it, ++i) 429 FillVAPicture(&slice_param.RefPicList1[i], *it); 430 431 return vaapi_wrapper_->SubmitBuffer(VASliceParameterBufferType, 432 sizeof(VASliceParameterBufferH264), 433 &slice_param); 434 } 435 436 bool VaapiH264Decoder::SendSliceData(const uint8* ptr, size_t size) { 437 // Can't help it, blame libva... 438 void* non_const_ptr = const_cast<uint8*>(ptr); 439 return vaapi_wrapper_->SubmitBuffer(VASliceDataBufferType, size, 440 non_const_ptr); 441 } 442 443 bool VaapiH264Decoder::PrepareRefPicLists(H264SliceHeader* slice_hdr) { 444 ref_pic_list0_.clear(); 445 ref_pic_list1_.clear(); 446 447 // Fill reference picture lists for B and S/SP slices. 448 if (slice_hdr->IsPSlice() || slice_hdr->IsSPSlice()) { 449 ConstructReferencePicListsP(slice_hdr); 450 return ModifyReferencePicList(slice_hdr, 0); 451 } 452 453 if (slice_hdr->IsBSlice()) { 454 ConstructReferencePicListsB(slice_hdr); 455 return ModifyReferencePicList(slice_hdr, 0) && 456 ModifyReferencePicList(slice_hdr, 1); 457 } 458 459 return true; 460 } 461 462 bool VaapiH264Decoder::QueueSlice(H264SliceHeader* slice_hdr) { 463 DCHECK(curr_pic_.get()); 464 465 if (!PrepareRefPicLists(slice_hdr)) 466 return false; 467 468 if (!SendVASliceParam(slice_hdr)) 469 return false; 470 471 if (!SendSliceData(slice_hdr->nalu_data, slice_hdr->nalu_size)) 472 return false; 473 474 return true; 475 } 476 477 // TODO(posciak) start using vaMapBuffer instead of vaCreateBuffer wherever 478 // possible. 479 bool VaapiH264Decoder::DecodePicture() { 480 DCHECK(curr_pic_.get()); 481 482 DVLOG(4) << "Decoding POC " << curr_pic_->pic_order_cnt; 483 DecodeSurface* dec_surface = DecodeSurfaceByPoC(curr_pic_->pic_order_cnt); 484 if (!dec_surface) { 485 DVLOG(1) << "Asked to decode an invalid POC " << curr_pic_->pic_order_cnt; 486 return false; 487 } 488 489 if (!vaapi_wrapper_->DecodeAndDestroyPendingBuffers( 490 dec_surface->va_surface()->id())) { 491 DVLOG(1) << "Failed decoding picture"; 492 return false; 493 } 494 495 return true; 496 } 497 498 499 bool VaapiH264Decoder::InitCurrPicture(H264SliceHeader* slice_hdr) { 500 DCHECK(curr_pic_.get()); 501 502 memset(curr_pic_.get(), 0, sizeof(H264Picture)); 503 504 curr_pic_->idr = slice_hdr->idr_pic_flag; 505 506 if (slice_hdr->field_pic_flag) { 507 curr_pic_->field = slice_hdr->bottom_field_flag ? H264Picture::FIELD_BOTTOM 508 : H264Picture::FIELD_TOP; 509 } else { 510 curr_pic_->field = H264Picture::FIELD_NONE; 511 } 512 513 curr_pic_->ref = slice_hdr->nal_ref_idc != 0; 514 // This assumes non-interlaced stream. 515 curr_pic_->frame_num = curr_pic_->pic_num = slice_hdr->frame_num; 516 517 if (!CalculatePicOrderCounts(slice_hdr)) 518 return false; 519 520 // Try to get an empty surface to decode this picture to. 521 if (!AssignSurfaceToPoC(curr_input_id_, curr_pic_->pic_order_cnt)) { 522 DVLOG(1) << "Failed getting a free surface for a picture"; 523 return false; 524 } 525 526 curr_pic_->long_term_reference_flag = slice_hdr->long_term_reference_flag; 527 curr_pic_->adaptive_ref_pic_marking_mode_flag = 528 slice_hdr->adaptive_ref_pic_marking_mode_flag; 529 530 // If the slice header indicates we will have to perform reference marking 531 // process after this picture is decoded, store required data for that 532 // purpose. 533 if (slice_hdr->adaptive_ref_pic_marking_mode_flag) { 534 COMPILE_ASSERT(sizeof(curr_pic_->ref_pic_marking) == 535 sizeof(slice_hdr->ref_pic_marking), 536 ref_pic_marking_array_sizes_do_not_match); 537 memcpy(curr_pic_->ref_pic_marking, slice_hdr->ref_pic_marking, 538 sizeof(curr_pic_->ref_pic_marking)); 539 } 540 541 return true; 542 } 543 544 bool VaapiH264Decoder::CalculatePicOrderCounts(H264SliceHeader* slice_hdr) { 545 DCHECK_NE(curr_sps_id_, -1); 546 const H264SPS* sps = parser_.GetSPS(curr_sps_id_); 547 548 int pic_order_cnt_lsb = slice_hdr->pic_order_cnt_lsb; 549 curr_pic_->pic_order_cnt_lsb = pic_order_cnt_lsb; 550 551 switch (sps->pic_order_cnt_type) { 552 case 0: 553 // See spec 8.2.1.1. 554 int prev_pic_order_cnt_msb, prev_pic_order_cnt_lsb; 555 if (slice_hdr->idr_pic_flag) { 556 prev_pic_order_cnt_msb = prev_pic_order_cnt_lsb = 0; 557 } else { 558 if (prev_ref_has_memmgmnt5_) { 559 if (prev_ref_field_ != H264Picture::FIELD_BOTTOM) { 560 prev_pic_order_cnt_msb = 0; 561 prev_pic_order_cnt_lsb = prev_ref_top_field_order_cnt_; 562 } else { 563 prev_pic_order_cnt_msb = 0; 564 prev_pic_order_cnt_lsb = 0; 565 } 566 } else { 567 prev_pic_order_cnt_msb = prev_ref_pic_order_cnt_msb_; 568 prev_pic_order_cnt_lsb = prev_ref_pic_order_cnt_lsb_; 569 } 570 } 571 572 DCHECK_NE(max_pic_order_cnt_lsb_, 0); 573 if ((pic_order_cnt_lsb < prev_pic_order_cnt_lsb) && 574 (prev_pic_order_cnt_lsb - pic_order_cnt_lsb >= 575 max_pic_order_cnt_lsb_ / 2)) { 576 curr_pic_->pic_order_cnt_msb = prev_pic_order_cnt_msb + 577 max_pic_order_cnt_lsb_; 578 } else if ((pic_order_cnt_lsb > prev_pic_order_cnt_lsb) && 579 (pic_order_cnt_lsb - prev_pic_order_cnt_lsb > 580 max_pic_order_cnt_lsb_ / 2)) { 581 curr_pic_->pic_order_cnt_msb = prev_pic_order_cnt_msb - 582 max_pic_order_cnt_lsb_; 583 } else { 584 curr_pic_->pic_order_cnt_msb = prev_pic_order_cnt_msb; 585 } 586 587 if (curr_pic_->field != H264Picture::FIELD_BOTTOM) { 588 curr_pic_->top_field_order_cnt = curr_pic_->pic_order_cnt_msb + 589 pic_order_cnt_lsb; 590 } 591 592 if (curr_pic_->field != H264Picture::FIELD_TOP) { 593 // TODO posciak: perhaps replace with pic->field? 594 if (!slice_hdr->field_pic_flag) { 595 curr_pic_->bottom_field_order_cnt = curr_pic_->top_field_order_cnt + 596 slice_hdr->delta_pic_order_cnt_bottom; 597 } else { 598 curr_pic_->bottom_field_order_cnt = curr_pic_->pic_order_cnt_msb + 599 pic_order_cnt_lsb; 600 } 601 } 602 break; 603 604 case 1: { 605 // See spec 8.2.1.2. 606 if (prev_has_memmgmnt5_) 607 prev_frame_num_offset_ = 0; 608 609 if (slice_hdr->idr_pic_flag) 610 curr_pic_->frame_num_offset = 0; 611 else if (prev_frame_num_ > slice_hdr->frame_num) 612 curr_pic_->frame_num_offset = prev_frame_num_offset_ + max_frame_num_; 613 else 614 curr_pic_->frame_num_offset = prev_frame_num_offset_; 615 616 int abs_frame_num = 0; 617 if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0) 618 abs_frame_num = curr_pic_->frame_num_offset + slice_hdr->frame_num; 619 else 620 abs_frame_num = 0; 621 622 if (slice_hdr->nal_ref_idc == 0 && abs_frame_num > 0) 623 --abs_frame_num; 624 625 int expected_pic_order_cnt = 0; 626 if (abs_frame_num > 0) { 627 if (sps->num_ref_frames_in_pic_order_cnt_cycle == 0) { 628 DVLOG(1) << "Invalid num_ref_frames_in_pic_order_cnt_cycle " 629 << "in stream"; 630 return false; 631 } 632 633 int pic_order_cnt_cycle_cnt = (abs_frame_num - 1) / 634 sps->num_ref_frames_in_pic_order_cnt_cycle; 635 int frame_num_in_pic_order_cnt_cycle = (abs_frame_num - 1) % 636 sps->num_ref_frames_in_pic_order_cnt_cycle; 637 638 expected_pic_order_cnt = pic_order_cnt_cycle_cnt * 639 sps->expected_delta_per_pic_order_cnt_cycle; 640 // frame_num_in_pic_order_cnt_cycle is verified < 255 in parser 641 for (int i = 0; i <= frame_num_in_pic_order_cnt_cycle; ++i) 642 expected_pic_order_cnt += sps->offset_for_ref_frame[i]; 643 } 644 645 if (!slice_hdr->nal_ref_idc) 646 expected_pic_order_cnt += sps->offset_for_non_ref_pic; 647 648 if (!slice_hdr->field_pic_flag) { 649 curr_pic_->top_field_order_cnt = expected_pic_order_cnt + 650 slice_hdr->delta_pic_order_cnt[0]; 651 curr_pic_->bottom_field_order_cnt = curr_pic_->top_field_order_cnt + 652 sps->offset_for_top_to_bottom_field + 653 slice_hdr->delta_pic_order_cnt[1]; 654 } else if (!slice_hdr->bottom_field_flag) { 655 curr_pic_->top_field_order_cnt = expected_pic_order_cnt + 656 slice_hdr->delta_pic_order_cnt[0]; 657 } else { 658 curr_pic_->bottom_field_order_cnt = expected_pic_order_cnt + 659 sps->offset_for_top_to_bottom_field + 660 slice_hdr->delta_pic_order_cnt[0]; 661 } 662 break; 663 } 664 665 case 2: 666 // See spec 8.2.1.3. 667 if (prev_has_memmgmnt5_) 668 prev_frame_num_offset_ = 0; 669 670 if (slice_hdr->idr_pic_flag) 671 curr_pic_->frame_num_offset = 0; 672 else if (prev_frame_num_ > slice_hdr->frame_num) 673 curr_pic_->frame_num_offset = prev_frame_num_offset_ + max_frame_num_; 674 else 675 curr_pic_->frame_num_offset = prev_frame_num_offset_; 676 677 int temp_pic_order_cnt; 678 if (slice_hdr->idr_pic_flag) { 679 temp_pic_order_cnt = 0; 680 } else if (!slice_hdr->nal_ref_idc) { 681 temp_pic_order_cnt = 682 2 * (curr_pic_->frame_num_offset + slice_hdr->frame_num) - 1; 683 } else { 684 temp_pic_order_cnt = 2 * (curr_pic_->frame_num_offset + 685 slice_hdr->frame_num); 686 } 687 688 if (!slice_hdr->field_pic_flag) { 689 curr_pic_->top_field_order_cnt = temp_pic_order_cnt; 690 curr_pic_->bottom_field_order_cnt = temp_pic_order_cnt; 691 } else if (slice_hdr->bottom_field_flag) { 692 curr_pic_->bottom_field_order_cnt = temp_pic_order_cnt; 693 } else { 694 curr_pic_->top_field_order_cnt = temp_pic_order_cnt; 695 } 696 break; 697 698 default: 699 DVLOG(1) << "Invalid pic_order_cnt_type: " << sps->pic_order_cnt_type; 700 return false; 701 } 702 703 switch (curr_pic_->field) { 704 case H264Picture::FIELD_NONE: 705 curr_pic_->pic_order_cnt = std::min(curr_pic_->top_field_order_cnt, 706 curr_pic_->bottom_field_order_cnt); 707 break; 708 case H264Picture::FIELD_TOP: 709 curr_pic_->pic_order_cnt = curr_pic_->top_field_order_cnt; 710 break; 711 case H264Picture::FIELD_BOTTOM: 712 curr_pic_->pic_order_cnt = curr_pic_->bottom_field_order_cnt; 713 break; 714 } 715 716 return true; 717 } 718 719 void VaapiH264Decoder::UpdatePicNums() { 720 for (H264DPB::Pictures::iterator it = dpb_.begin(); it != dpb_.end(); ++it) { 721 H264Picture* pic = *it; 722 DCHECK(pic); 723 if (!pic->ref) 724 continue; 725 726 // Below assumes non-interlaced stream. 727 DCHECK_EQ(pic->field, H264Picture::FIELD_NONE); 728 if (pic->long_term) { 729 pic->long_term_pic_num = pic->long_term_frame_idx; 730 } else { 731 if (pic->frame_num > frame_num_) 732 pic->frame_num_wrap = pic->frame_num - max_frame_num_; 733 else 734 pic->frame_num_wrap = pic->frame_num; 735 736 pic->pic_num = pic->frame_num_wrap; 737 } 738 } 739 } 740 741 struct PicNumDescCompare { 742 bool operator()(const H264Picture* a, const H264Picture* b) const { 743 return a->pic_num > b->pic_num; 744 } 745 }; 746 747 struct LongTermPicNumAscCompare { 748 bool operator()(const H264Picture* a, const H264Picture* b) const { 749 return a->long_term_pic_num < b->long_term_pic_num; 750 } 751 }; 752 753 void VaapiH264Decoder::ConstructReferencePicListsP(H264SliceHeader* slice_hdr) { 754 // RefPicList0 (8.2.4.2.1) [[1] [2]], where: 755 // [1] shortterm ref pics sorted by descending pic_num, 756 // [2] longterm ref pics by ascending long_term_pic_num. 757 DCHECK(ref_pic_list0_.empty() && ref_pic_list1_.empty()); 758 // First get the short ref pics... 759 dpb_.GetShortTermRefPicsAppending(ref_pic_list0_); 760 size_t num_short_refs = ref_pic_list0_.size(); 761 762 // and sort them to get [1]. 763 std::sort(ref_pic_list0_.begin(), ref_pic_list0_.end(), PicNumDescCompare()); 764 765 // Now get long term pics and sort them by long_term_pic_num to get [2]. 766 dpb_.GetLongTermRefPicsAppending(ref_pic_list0_); 767 std::sort(ref_pic_list0_.begin() + num_short_refs, ref_pic_list0_.end(), 768 LongTermPicNumAscCompare()); 769 770 // Cut off if we have more than requested in slice header. 771 ref_pic_list0_.resize(slice_hdr->num_ref_idx_l0_active_minus1 + 1); 772 } 773 774 struct POCAscCompare { 775 bool operator()(const H264Picture* a, const H264Picture* b) const { 776 return a->pic_order_cnt < b->pic_order_cnt; 777 } 778 }; 779 780 struct POCDescCompare { 781 bool operator()(const H264Picture* a, const H264Picture* b) const { 782 return a->pic_order_cnt > b->pic_order_cnt; 783 } 784 }; 785 786 void VaapiH264Decoder::ConstructReferencePicListsB(H264SliceHeader* slice_hdr) { 787 // RefPicList0 (8.2.4.2.3) [[1] [2] [3]], where: 788 // [1] shortterm ref pics with POC < curr_pic's POC sorted by descending POC, 789 // [2] shortterm ref pics with POC > curr_pic's POC by ascending POC, 790 // [3] longterm ref pics by ascending long_term_pic_num. 791 DCHECK(ref_pic_list0_.empty() && ref_pic_list1_.empty()); 792 dpb_.GetShortTermRefPicsAppending(ref_pic_list0_); 793 size_t num_short_refs = ref_pic_list0_.size(); 794 795 // First sort ascending, this will put [1] in right place and finish [2]. 796 std::sort(ref_pic_list0_.begin(), ref_pic_list0_.end(), POCAscCompare()); 797 798 // Find first with POC > curr_pic's POC to get first element in [2]... 799 H264Picture::PtrVector::iterator iter; 800 iter = std::upper_bound(ref_pic_list0_.begin(), ref_pic_list0_.end(), 801 curr_pic_.get(), POCAscCompare()); 802 803 // and sort [1] descending, thus finishing sequence [1] [2]. 804 std::sort(ref_pic_list0_.begin(), iter, POCDescCompare()); 805 806 // Now add [3] and sort by ascending long_term_pic_num. 807 dpb_.GetLongTermRefPicsAppending(ref_pic_list0_); 808 std::sort(ref_pic_list0_.begin() + num_short_refs, ref_pic_list0_.end(), 809 LongTermPicNumAscCompare()); 810 811 // RefPicList1 (8.2.4.2.4) [[1] [2] [3]], where: 812 // [1] shortterm ref pics with POC > curr_pic's POC sorted by ascending POC, 813 // [2] shortterm ref pics with POC < curr_pic's POC by descending POC, 814 // [3] longterm ref pics by ascending long_term_pic_num. 815 816 dpb_.GetShortTermRefPicsAppending(ref_pic_list1_); 817 num_short_refs = ref_pic_list1_.size(); 818 819 // First sort by descending POC. 820 std::sort(ref_pic_list1_.begin(), ref_pic_list1_.end(), POCDescCompare()); 821 822 // Find first with POC < curr_pic's POC to get first element in [2]... 823 iter = std::upper_bound(ref_pic_list1_.begin(), ref_pic_list1_.end(), 824 curr_pic_.get(), POCDescCompare()); 825 826 // and sort [1] ascending. 827 std::sort(ref_pic_list1_.begin(), iter, POCAscCompare()); 828 829 // Now add [3] and sort by ascending long_term_pic_num 830 dpb_.GetShortTermRefPicsAppending(ref_pic_list1_); 831 std::sort(ref_pic_list1_.begin() + num_short_refs, ref_pic_list1_.end(), 832 LongTermPicNumAscCompare()); 833 834 // If lists identical, swap first two entries in RefPicList1 (spec 8.2.4.2.3) 835 if (ref_pic_list1_.size() > 1 && 836 std::equal(ref_pic_list0_.begin(), ref_pic_list0_.end(), 837 ref_pic_list1_.begin())) 838 std::swap(ref_pic_list1_[0], ref_pic_list1_[1]); 839 840 // Per 8.2.4.2 it's possible for num_ref_idx_lX_active_minus1 to indicate 841 // there should be more ref pics on list than we constructed. 842 // Those superfluous ones should be treated as non-reference. 843 ref_pic_list0_.resize(slice_hdr->num_ref_idx_l0_active_minus1 + 1); 844 ref_pic_list1_.resize(slice_hdr->num_ref_idx_l1_active_minus1 + 1); 845 } 846 847 // See 8.2.4 848 int VaapiH264Decoder::PicNumF(H264Picture *pic) { 849 if (!pic) 850 return -1; 851 852 if (!pic->long_term) 853 return pic->pic_num; 854 else 855 return max_pic_num_; 856 } 857 858 // See 8.2.4 859 int VaapiH264Decoder::LongTermPicNumF(H264Picture *pic) { 860 if (pic->ref && pic->long_term) 861 return pic->long_term_pic_num; 862 else 863 return 2 * (max_long_term_frame_idx_ + 1); 864 } 865 866 // Shift elements on the |v| starting from |from| to |to|, inclusive, 867 // one position to the right and insert pic at |from|. 868 static void ShiftRightAndInsert(H264Picture::PtrVector *v, 869 int from, 870 int to, 871 H264Picture* pic) { 872 // Security checks, do not disable in Debug mode. 873 CHECK(from <= to); 874 CHECK(to <= std::numeric_limits<int>::max() - 2); 875 // Additional checks. Debug mode ok. 876 DCHECK(v); 877 DCHECK(pic); 878 DCHECK((to + 1 == static_cast<int>(v->size())) || 879 (to + 2 == static_cast<int>(v->size()))); 880 881 v->resize(to + 2); 882 883 for (int i = to + 1; i > from; --i) 884 (*v)[i] = (*v)[i - 1]; 885 886 (*v)[from] = pic; 887 } 888 889 bool VaapiH264Decoder::ModifyReferencePicList(H264SliceHeader *slice_hdr, 890 int list) { 891 int num_ref_idx_lX_active_minus1; 892 H264Picture::PtrVector* ref_pic_listx; 893 H264ModificationOfPicNum* list_mod; 894 895 // This can process either ref_pic_list0 or ref_pic_list1, depending on 896 // the list argument. Set up pointers to proper list to be processed here. 897 if (list == 0) { 898 if (!slice_hdr->ref_pic_list_modification_flag_l0) 899 return true; 900 901 list_mod = slice_hdr->ref_list_l0_modifications; 902 num_ref_idx_lX_active_minus1 = ref_pic_list0_.size() - 1; 903 904 ref_pic_listx = &ref_pic_list0_; 905 } else { 906 if (!slice_hdr->ref_pic_list_modification_flag_l1) 907 return true; 908 909 list_mod = slice_hdr->ref_list_l1_modifications; 910 num_ref_idx_lX_active_minus1 = ref_pic_list1_.size() - 1; 911 912 ref_pic_listx = &ref_pic_list1_; 913 } 914 915 DCHECK_GE(num_ref_idx_lX_active_minus1, 0); 916 917 // Spec 8.2.4.3: 918 // Reorder pictures on the list in a way specified in the stream. 919 int pic_num_lx_pred = curr_pic_->pic_num; 920 int ref_idx_lx = 0; 921 int pic_num_lx_no_wrap; 922 int pic_num_lx; 923 bool done = false; 924 H264Picture* pic; 925 for (int i = 0; i < H264SliceHeader::kRefListModSize && !done; ++i) { 926 switch (list_mod->modification_of_pic_nums_idc) { 927 case 0: 928 case 1: 929 // Modify short reference picture position. 930 if (list_mod->modification_of_pic_nums_idc == 0) { 931 // Subtract given value from predicted PicNum. 932 pic_num_lx_no_wrap = pic_num_lx_pred - 933 (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1); 934 // Wrap around max_pic_num_ if it becomes < 0 as result 935 // of subtraction. 936 if (pic_num_lx_no_wrap < 0) 937 pic_num_lx_no_wrap += max_pic_num_; 938 } else { 939 // Add given value to predicted PicNum. 940 pic_num_lx_no_wrap = pic_num_lx_pred + 941 (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1); 942 // Wrap around max_pic_num_ if it becomes >= max_pic_num_ as result 943 // of the addition. 944 if (pic_num_lx_no_wrap >= max_pic_num_) 945 pic_num_lx_no_wrap -= max_pic_num_; 946 } 947 948 // For use in next iteration. 949 pic_num_lx_pred = pic_num_lx_no_wrap; 950 951 if (pic_num_lx_no_wrap > curr_pic_->pic_num) 952 pic_num_lx = pic_num_lx_no_wrap - max_pic_num_; 953 else 954 pic_num_lx = pic_num_lx_no_wrap; 955 956 DCHECK_LT(num_ref_idx_lX_active_minus1 + 1, 957 H264SliceHeader::kRefListModSize); 958 pic = dpb_.GetShortRefPicByPicNum(pic_num_lx); 959 if (!pic) { 960 DVLOG(1) << "Malformed stream, no pic num " << pic_num_lx; 961 return false; 962 } 963 ShiftRightAndInsert(ref_pic_listx, ref_idx_lx, 964 num_ref_idx_lX_active_minus1, pic); 965 ref_idx_lx++; 966 967 for (int src = ref_idx_lx, dst = ref_idx_lx; 968 src <= num_ref_idx_lX_active_minus1 + 1; ++src) { 969 if (PicNumF((*ref_pic_listx)[src]) != pic_num_lx) 970 (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src]; 971 } 972 break; 973 974 case 2: 975 // Modify long term reference picture position. 976 DCHECK_LT(num_ref_idx_lX_active_minus1 + 1, 977 H264SliceHeader::kRefListModSize); 978 pic = dpb_.GetLongRefPicByLongTermPicNum(list_mod->long_term_pic_num); 979 if (!pic) { 980 DVLOG(1) << "Malformed stream, no pic num " 981 << list_mod->long_term_pic_num; 982 return false; 983 } 984 ShiftRightAndInsert(ref_pic_listx, ref_idx_lx, 985 num_ref_idx_lX_active_minus1, pic); 986 ref_idx_lx++; 987 988 for (int src = ref_idx_lx, dst = ref_idx_lx; 989 src <= num_ref_idx_lX_active_minus1 + 1; ++src) { 990 if (LongTermPicNumF((*ref_pic_listx)[src]) 991 != static_cast<int>(list_mod->long_term_pic_num)) 992 (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src]; 993 } 994 break; 995 996 case 3: 997 // End of modification list. 998 done = true; 999 break; 1000 1001 default: 1002 // May be recoverable. 1003 DVLOG(1) << "Invalid modification_of_pic_nums_idc=" 1004 << list_mod->modification_of_pic_nums_idc 1005 << " in position " << i; 1006 break; 1007 } 1008 1009 ++list_mod; 1010 } 1011 1012 // Per NOTE 2 in 8.2.4.3.2, the ref_pic_listx size in the above loop is 1013 // temporarily made one element longer than the required final list. 1014 // Resize the list back to its required size. 1015 ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1); 1016 1017 return true; 1018 } 1019 1020 bool VaapiH264Decoder::OutputPic(H264Picture* pic) { 1021 DCHECK(!pic->outputted); 1022 pic->outputted = true; 1023 last_output_poc_ = pic->pic_order_cnt; 1024 1025 DecodeSurface* dec_surface = DecodeSurfaceByPoC(pic->pic_order_cnt); 1026 if (!dec_surface) 1027 return false; 1028 1029 DCHECK_GE(dec_surface->input_id(), 0); 1030 DVLOG(4) << "Posting output task for POC: " << pic->pic_order_cnt 1031 << " input_id: " << dec_surface->input_id(); 1032 output_pic_cb_.Run(dec_surface->input_id(), dec_surface->va_surface()); 1033 1034 return true; 1035 } 1036 1037 void VaapiH264Decoder::ClearDPB() { 1038 // Clear DPB contents, marking the pictures as unused first. 1039 for (H264DPB::Pictures::iterator it = dpb_.begin(); it != dpb_.end(); ++it) 1040 UnassignSurfaceFromPoC((*it)->pic_order_cnt); 1041 1042 dpb_.Clear(); 1043 last_output_poc_ = 0; 1044 } 1045 1046 bool VaapiH264Decoder::OutputAllRemainingPics() { 1047 // Output all pictures that are waiting to be outputted. 1048 FinishPrevFrameIfPresent(); 1049 H264Picture::PtrVector to_output; 1050 dpb_.GetNotOutputtedPicsAppending(to_output); 1051 // Sort them by ascending POC to output in order. 1052 std::sort(to_output.begin(), to_output.end(), POCAscCompare()); 1053 1054 H264Picture::PtrVector::iterator it; 1055 for (it = to_output.begin(); it != to_output.end(); ++it) { 1056 if (!OutputPic(*it)) { 1057 DVLOG(1) << "Failed to output pic POC: " << (*it)->pic_order_cnt; 1058 return false; 1059 } 1060 } 1061 1062 return true; 1063 } 1064 1065 bool VaapiH264Decoder::Flush() { 1066 DVLOG(2) << "Decoder flush"; 1067 1068 if (!OutputAllRemainingPics()) 1069 return false; 1070 1071 ClearDPB(); 1072 1073 DCHECK(decode_surfaces_in_use_.empty()); 1074 return true; 1075 } 1076 1077 bool VaapiH264Decoder::StartNewFrame(H264SliceHeader* slice_hdr) { 1078 // TODO posciak: add handling of max_num_ref_frames per spec. 1079 1080 // If the new frame is an IDR, output what's left to output and clear DPB 1081 if (slice_hdr->idr_pic_flag) { 1082 // (unless we are explicitly instructed not to do so). 1083 if (!slice_hdr->no_output_of_prior_pics_flag) { 1084 // Output DPB contents. 1085 if (!Flush()) 1086 return false; 1087 } 1088 dpb_.Clear(); 1089 last_output_poc_ = 0; 1090 } 1091 1092 // curr_pic_ should have either been added to DPB or discarded when finishing 1093 // the last frame. DPB is responsible for releasing that memory once it's 1094 // not needed anymore. 1095 DCHECK(!curr_pic_.get()); 1096 curr_pic_.reset(new H264Picture); 1097 CHECK(curr_pic_.get()); 1098 1099 if (!InitCurrPicture(slice_hdr)) 1100 return false; 1101 1102 DCHECK_GT(max_frame_num_, 0); 1103 1104 UpdatePicNums(); 1105 1106 // Send parameter buffers before each new picture, before the first slice. 1107 if (!SendPPS()) 1108 return false; 1109 1110 if (!SendIQMatrix()) 1111 return false; 1112 1113 if (!QueueSlice(slice_hdr)) 1114 return false; 1115 1116 return true; 1117 } 1118 1119 bool VaapiH264Decoder::HandleMemoryManagementOps() { 1120 // 8.2.5.4 1121 for (unsigned int i = 0; i < arraysize(curr_pic_->ref_pic_marking); ++i) { 1122 // Code below does not support interlaced stream (per-field pictures). 1123 H264DecRefPicMarking* ref_pic_marking = &curr_pic_->ref_pic_marking[i]; 1124 H264Picture* to_mark; 1125 int pic_num_x; 1126 1127 switch (ref_pic_marking->memory_mgmnt_control_operation) { 1128 case 0: 1129 // Normal end of operations' specification. 1130 return true; 1131 1132 case 1: 1133 // Mark a short term reference picture as unused so it can be removed 1134 // if outputted. 1135 pic_num_x = curr_pic_->pic_num - 1136 (ref_pic_marking->difference_of_pic_nums_minus1 + 1); 1137 to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x); 1138 if (to_mark) { 1139 to_mark->ref = false; 1140 } else { 1141 DVLOG(1) << "Invalid short ref pic num to unmark"; 1142 return false; 1143 } 1144 break; 1145 1146 case 2: 1147 // Mark a long term reference picture as unused so it can be removed 1148 // if outputted. 1149 to_mark = dpb_.GetLongRefPicByLongTermPicNum( 1150 ref_pic_marking->long_term_pic_num); 1151 if (to_mark) { 1152 to_mark->ref = false; 1153 } else { 1154 DVLOG(1) << "Invalid long term ref pic num to unmark"; 1155 return false; 1156 } 1157 break; 1158 1159 case 3: 1160 // Mark a short term reference picture as long term reference. 1161 pic_num_x = curr_pic_->pic_num - 1162 (ref_pic_marking->difference_of_pic_nums_minus1 + 1); 1163 to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x); 1164 if (to_mark) { 1165 DCHECK(to_mark->ref && !to_mark->long_term); 1166 to_mark->long_term = true; 1167 to_mark->long_term_frame_idx = ref_pic_marking->long_term_frame_idx; 1168 } else { 1169 DVLOG(1) << "Invalid short term ref pic num to mark as long ref"; 1170 return false; 1171 } 1172 break; 1173 1174 case 4: { 1175 // Unmark all reference pictures with long_term_frame_idx over new max. 1176 max_long_term_frame_idx_ 1177 = ref_pic_marking->max_long_term_frame_idx_plus1 - 1; 1178 H264Picture::PtrVector long_terms; 1179 dpb_.GetLongTermRefPicsAppending(long_terms); 1180 for (size_t i = 0; i < long_terms.size(); ++i) { 1181 H264Picture* pic = long_terms[i]; 1182 DCHECK(pic->ref && pic->long_term); 1183 // Ok to cast, max_long_term_frame_idx is much smaller than 16bit. 1184 if (pic->long_term_frame_idx > 1185 static_cast<int>(max_long_term_frame_idx_)) 1186 pic->ref = false; 1187 } 1188 break; 1189 } 1190 1191 case 5: 1192 // Unmark all reference pictures. 1193 dpb_.MarkAllUnusedForRef(); 1194 max_long_term_frame_idx_ = -1; 1195 curr_pic_->mem_mgmt_5 = true; 1196 break; 1197 1198 case 6: { 1199 // Replace long term reference pictures with current picture. 1200 // First unmark if any existing with this long_term_frame_idx... 1201 H264Picture::PtrVector long_terms; 1202 dpb_.GetLongTermRefPicsAppending(long_terms); 1203 for (size_t i = 0; i < long_terms.size(); ++i) { 1204 H264Picture* pic = long_terms[i]; 1205 DCHECK(pic->ref && pic->long_term); 1206 // Ok to cast, long_term_frame_idx is much smaller than 16bit. 1207 if (pic->long_term_frame_idx == 1208 static_cast<int>(ref_pic_marking->long_term_frame_idx)) 1209 pic->ref = false; 1210 } 1211 1212 // and mark the current one instead. 1213 curr_pic_->ref = true; 1214 curr_pic_->long_term = true; 1215 curr_pic_->long_term_frame_idx = ref_pic_marking->long_term_frame_idx; 1216 break; 1217 } 1218 1219 default: 1220 // Would indicate a bug in parser. 1221 NOTREACHED(); 1222 } 1223 } 1224 1225 return true; 1226 } 1227 1228 // This method ensures that DPB does not overflow, either by removing 1229 // reference pictures as specified in the stream, or using a sliding window 1230 // procedure to remove the oldest one. 1231 // It also performs marking and unmarking pictures as reference. 1232 // See spac 8.2.5.1. 1233 void VaapiH264Decoder::ReferencePictureMarking() { 1234 if (curr_pic_->idr) { 1235 // If current picture is an IDR, all reference pictures are unmarked. 1236 dpb_.MarkAllUnusedForRef(); 1237 1238 if (curr_pic_->long_term_reference_flag) { 1239 curr_pic_->long_term = true; 1240 curr_pic_->long_term_frame_idx = 0; 1241 max_long_term_frame_idx_ = 0; 1242 } else { 1243 curr_pic_->long_term = false; 1244 max_long_term_frame_idx_ = -1; 1245 } 1246 } else { 1247 if (!curr_pic_->adaptive_ref_pic_marking_mode_flag) { 1248 // If non-IDR, and the stream does not indicate what we should do to 1249 // ensure DPB doesn't overflow, discard oldest picture. 1250 // See spec 8.2.5.3. 1251 if (curr_pic_->field == H264Picture::FIELD_NONE) { 1252 DCHECK_LE(dpb_.CountRefPics(), 1253 std::max<int>(parser_.GetSPS(curr_sps_id_)->max_num_ref_frames, 1254 1)); 1255 if (dpb_.CountRefPics() == 1256 std::max<int>(parser_.GetSPS(curr_sps_id_)->max_num_ref_frames, 1257 1)) { 1258 // Max number of reference pics reached, 1259 // need to remove one of the short term ones. 1260 // Find smallest frame_num_wrap short reference picture and mark 1261 // it as unused. 1262 H264Picture* to_unmark = dpb_.GetLowestFrameNumWrapShortRefPic(); 1263 if (to_unmark == NULL) { 1264 DVLOG(1) << "Couldn't find a short ref picture to unmark"; 1265 return; 1266 } 1267 to_unmark->ref = false; 1268 } 1269 } else { 1270 // Shouldn't get here. 1271 DVLOG(1) << "Interlaced video not supported."; 1272 report_error_to_uma_cb_.Run(INTERLACED_STREAM); 1273 } 1274 } else { 1275 // Stream has instructions how to discard pictures from DPB and how 1276 // to mark/unmark existing reference pictures. Do it. 1277 // Spec 8.2.5.4. 1278 if (curr_pic_->field == H264Picture::FIELD_NONE) { 1279 HandleMemoryManagementOps(); 1280 } else { 1281 // Shouldn't get here. 1282 DVLOG(1) << "Interlaced video not supported."; 1283 report_error_to_uma_cb_.Run(INTERLACED_STREAM); 1284 } 1285 } 1286 } 1287 } 1288 1289 bool VaapiH264Decoder::FinishPicture() { 1290 DCHECK(curr_pic_.get()); 1291 1292 // Finish processing previous picture. 1293 // Start by storing previous reference picture data for later use, 1294 // if picture being finished is a reference picture. 1295 if (curr_pic_->ref) { 1296 ReferencePictureMarking(); 1297 prev_ref_has_memmgmnt5_ = curr_pic_->mem_mgmt_5; 1298 prev_ref_top_field_order_cnt_ = curr_pic_->top_field_order_cnt; 1299 prev_ref_pic_order_cnt_msb_ = curr_pic_->pic_order_cnt_msb; 1300 prev_ref_pic_order_cnt_lsb_ = curr_pic_->pic_order_cnt_lsb; 1301 prev_ref_field_ = curr_pic_->field; 1302 } 1303 prev_has_memmgmnt5_ = curr_pic_->mem_mgmt_5; 1304 prev_frame_num_offset_ = curr_pic_->frame_num_offset; 1305 1306 // Remove unused (for reference or later output) pictures from DPB, marking 1307 // them as such. 1308 for (H264DPB::Pictures::iterator it = dpb_.begin(); it != dpb_.end(); ++it) { 1309 if ((*it)->outputted && !(*it)->ref) 1310 UnassignSurfaceFromPoC((*it)->pic_order_cnt); 1311 } 1312 dpb_.DeleteUnused(); 1313 1314 DVLOG(4) << "Finishing picture, entries in DPB: " << dpb_.size(); 1315 1316 // Whatever happens below, curr_pic_ will stop managing the pointer to the 1317 // picture after this function returns. The ownership will either be 1318 // transferred to DPB, if the image is still needed (for output and/or 1319 // reference), or the memory will be released if we manage to output it here 1320 // without having to store it for future reference. 1321 scoped_ptr<H264Picture> pic(curr_pic_.release()); 1322 1323 // Get all pictures that haven't been outputted yet. 1324 H264Picture::PtrVector not_outputted; 1325 // TODO(posciak): pass as pointer, not reference (violates coding style). 1326 dpb_.GetNotOutputtedPicsAppending(not_outputted); 1327 // Include the one we've just decoded. 1328 not_outputted.push_back(pic.get()); 1329 // Sort in output order. 1330 std::sort(not_outputted.begin(), not_outputted.end(), POCAscCompare()); 1331 1332 // Try to output as many pictures as we can. A picture can be output 1333 // if its POC is next after the previously outputted one (which means 1334 // last_output_poc_ + 2, because POCs are incremented by 2 to accommodate 1335 // fields when decoding interleaved streams). POC can also be equal to 1336 // last outputted picture's POC when it wraps around back to 0. 1337 // If the outputted picture is not a reference picture, it doesn't have 1338 // to remain in the DPB and can be removed. 1339 H264Picture::PtrVector::iterator output_candidate = not_outputted.begin(); 1340 for (; output_candidate != not_outputted.end() && 1341 (*output_candidate)->pic_order_cnt <= last_output_poc_ + 2; 1342 ++output_candidate) { 1343 int poc = (*output_candidate)->pic_order_cnt; 1344 DCHECK_GE(poc, last_output_poc_); 1345 if (!OutputPic(*output_candidate)) 1346 return false; 1347 1348 if (!(*output_candidate)->ref) { 1349 // Current picture hasn't been inserted into DPB yet, so don't remove it 1350 // if we managed to output it immediately. 1351 if (*output_candidate != pic) 1352 dpb_.DeleteByPOC(poc); 1353 // Mark as unused. 1354 UnassignSurfaceFromPoC(poc); 1355 } 1356 } 1357 1358 // If we haven't managed to output the picture that we just decoded, or if 1359 // it's a reference picture, we have to store it in DPB. 1360 if (!pic->outputted || pic->ref) { 1361 if (dpb_.IsFull()) { 1362 // If we haven't managed to output anything to free up space in DPB 1363 // to store this picture, it's an error in the stream. 1364 DVLOG(1) << "Could not free up space in DPB!"; 1365 return false; 1366 } 1367 1368 dpb_.StorePic(pic.release()); 1369 } 1370 1371 return true; 1372 } 1373 1374 static int LevelToMaxDpbMbs(int level) { 1375 // See table A-1 in spec. 1376 switch (level) { 1377 case 10: return 396; 1378 case 11: return 900; 1379 case 12: // fallthrough 1380 case 13: // fallthrough 1381 case 20: return 2376; 1382 case 21: return 4752; 1383 case 22: // fallthrough 1384 case 30: return 8100; 1385 case 31: return 18000; 1386 case 32: return 20480; 1387 case 40: // fallthrough 1388 case 41: return 32768; 1389 case 42: return 34816; 1390 case 50: return 110400; 1391 case 51: // fallthrough 1392 case 52: return 184320; 1393 default: 1394 DVLOG(1) << "Invalid codec level (" << level << ")"; 1395 return 0; 1396 } 1397 } 1398 1399 bool VaapiH264Decoder::ProcessSPS(int sps_id, bool* need_new_buffers) { 1400 const H264SPS* sps = parser_.GetSPS(sps_id); 1401 DCHECK(sps); 1402 DVLOG(4) << "Processing SPS"; 1403 1404 *need_new_buffers = false; 1405 1406 if (sps->frame_mbs_only_flag == 0) { 1407 DVLOG(1) << "frame_mbs_only_flag != 1 not supported"; 1408 report_error_to_uma_cb_.Run(FRAME_MBS_ONLY_FLAG_NOT_ONE); 1409 return false; 1410 } 1411 1412 if (sps->gaps_in_frame_num_value_allowed_flag) { 1413 DVLOG(1) << "Gaps in frame numbers not supported"; 1414 report_error_to_uma_cb_.Run(GAPS_IN_FRAME_NUM); 1415 return false; 1416 } 1417 1418 curr_sps_id_ = sps->seq_parameter_set_id; 1419 1420 // Calculate picture height/width in macroblocks and pixels 1421 // (spec 7.4.2.1.1, 7.4.3). 1422 int width_mb = sps->pic_width_in_mbs_minus1 + 1; 1423 int height_mb = (2 - sps->frame_mbs_only_flag) * 1424 (sps->pic_height_in_map_units_minus1 + 1); 1425 1426 gfx::Size new_pic_size(16 * width_mb, 16 * height_mb); 1427 if (new_pic_size.IsEmpty()) { 1428 DVLOG(1) << "Invalid picture size: " << new_pic_size.ToString(); 1429 return false; 1430 } 1431 1432 if (!pic_size_.IsEmpty() && new_pic_size == pic_size_) { 1433 // Already have surfaces and this SPS keeps the same resolution, 1434 // no need to request a new set. 1435 return true; 1436 } 1437 1438 pic_size_ = new_pic_size; 1439 DVLOG(1) << "New picture size: " << pic_size_.ToString(); 1440 1441 max_pic_order_cnt_lsb_ = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4); 1442 max_frame_num_ = 1 << (sps->log2_max_frame_num_minus4 + 4); 1443 1444 int level = sps->level_idc; 1445 int max_dpb_mbs = LevelToMaxDpbMbs(level); 1446 if (max_dpb_mbs == 0) 1447 return false; 1448 1449 size_t max_dpb_size = std::min(max_dpb_mbs / (width_mb * height_mb), 1450 static_cast<int>(H264DPB::kDPBMaxSize)); 1451 DVLOG(1) << "Codec level: " << level << ", DPB size: " << max_dpb_size; 1452 if (max_dpb_size == 0) { 1453 DVLOG(1) << "Invalid DPB Size"; 1454 return false; 1455 } 1456 1457 dpb_.set_max_num_pics(max_dpb_size); 1458 1459 *need_new_buffers = true; 1460 return true; 1461 } 1462 1463 bool VaapiH264Decoder::ProcessPPS(int pps_id) { 1464 const H264PPS* pps = parser_.GetPPS(pps_id); 1465 DCHECK(pps); 1466 1467 curr_pps_id_ = pps->pic_parameter_set_id; 1468 1469 return true; 1470 } 1471 1472 bool VaapiH264Decoder::FinishPrevFrameIfPresent() { 1473 // If we already have a frame waiting to be decoded, decode it and finish. 1474 if (curr_pic_ != NULL) { 1475 if (!DecodePicture()) 1476 return false; 1477 return FinishPicture(); 1478 } 1479 1480 return true; 1481 } 1482 1483 bool VaapiH264Decoder::ProcessSlice(H264SliceHeader* slice_hdr) { 1484 prev_frame_num_ = frame_num_; 1485 frame_num_ = slice_hdr->frame_num; 1486 1487 if (prev_frame_num_ > 0 && prev_frame_num_ < frame_num_ - 1) { 1488 DVLOG(1) << "Gap in frame_num!"; 1489 report_error_to_uma_cb_.Run(GAPS_IN_FRAME_NUM); 1490 return false; 1491 } 1492 1493 if (slice_hdr->field_pic_flag == 0) 1494 max_pic_num_ = max_frame_num_; 1495 else 1496 max_pic_num_ = 2 * max_frame_num_; 1497 1498 // TODO posciak: switch to new picture detection per 7.4.1.2.4. 1499 if (curr_pic_ != NULL && slice_hdr->first_mb_in_slice != 0) { 1500 // This is just some more slice data of the current picture, so 1501 // just queue it and return. 1502 QueueSlice(slice_hdr); 1503 return true; 1504 } else { 1505 // A new frame, so first finish the previous one before processing it... 1506 if (!FinishPrevFrameIfPresent()) 1507 return false; 1508 1509 // and then start a new one. 1510 return StartNewFrame(slice_hdr); 1511 } 1512 } 1513 1514 #define SET_ERROR_AND_RETURN() \ 1515 do { \ 1516 DVLOG(1) << "Error during decode"; \ 1517 state_ = kError; \ 1518 return VaapiH264Decoder::kDecodeError; \ 1519 } while (0) 1520 1521 void VaapiH264Decoder::SetStream(uint8* ptr, size_t size, int32 input_id) { 1522 DCHECK(ptr); 1523 DCHECK(size); 1524 1525 // Got new input stream data from the client. 1526 DVLOG(4) << "New input stream id: " << input_id << " at: " << (void*) ptr 1527 << " size: " << size; 1528 parser_.SetStream(ptr, size); 1529 curr_input_id_ = input_id; 1530 } 1531 1532 VaapiH264Decoder::DecResult VaapiH264Decoder::Decode() { 1533 H264Parser::Result par_res; 1534 H264NALU nalu; 1535 DCHECK_NE(state_, kError); 1536 1537 while (1) { 1538 // If we've already decoded some of the stream (after reset, i.e. we are 1539 // not in kNeedStreamMetadata state), we may be able to go back into 1540 // decoding state not only starting at/resuming from an SPS, but also from 1541 // other resume points, such as IDRs. In the latter case we need an output 1542 // surface, because we will end up decoding that IDR in the process. 1543 // Otherwise we just look for an SPS and don't produce any output frames. 1544 if (state_ != kNeedStreamMetadata && available_va_surfaces_.empty()) { 1545 DVLOG(4) << "No output surfaces available"; 1546 return kRanOutOfSurfaces; 1547 } 1548 1549 par_res = parser_.AdvanceToNextNALU(&nalu); 1550 if (par_res == H264Parser::kEOStream) 1551 return kRanOutOfStreamData; 1552 else if (par_res != H264Parser::kOk) 1553 SET_ERROR_AND_RETURN(); 1554 1555 DVLOG(4) << "NALU found: " << static_cast<int>(nalu.nal_unit_type); 1556 1557 switch (nalu.nal_unit_type) { 1558 case H264NALU::kNonIDRSlice: 1559 // We can't resume from a non-IDR slice. 1560 if (state_ != kDecoding) 1561 break; 1562 // else fallthrough 1563 case H264NALU::kIDRSlice: { 1564 // TODO(posciak): the IDR may require an SPS that we don't have 1565 // available. For now we'd fail if that happens, but ideally we'd like 1566 // to keep going until the next SPS in the stream. 1567 if (state_ == kNeedStreamMetadata) { 1568 // We need an SPS, skip this IDR and keep looking. 1569 break; 1570 } 1571 1572 // If after reset, we should be able to recover from an IDR. 1573 H264SliceHeader slice_hdr; 1574 1575 par_res = parser_.ParseSliceHeader(nalu, &slice_hdr); 1576 if (par_res != H264Parser::kOk) 1577 SET_ERROR_AND_RETURN(); 1578 1579 if (!ProcessSlice(&slice_hdr)) 1580 SET_ERROR_AND_RETURN(); 1581 1582 state_ = kDecoding; 1583 break; 1584 } 1585 1586 case H264NALU::kSPS: { 1587 int sps_id; 1588 1589 if (!FinishPrevFrameIfPresent()) 1590 SET_ERROR_AND_RETURN(); 1591 1592 par_res = parser_.ParseSPS(&sps_id); 1593 if (par_res != H264Parser::kOk) 1594 SET_ERROR_AND_RETURN(); 1595 1596 bool need_new_buffers = false; 1597 if (!ProcessSPS(sps_id, &need_new_buffers)) 1598 SET_ERROR_AND_RETURN(); 1599 1600 state_ = kDecoding; 1601 1602 if (need_new_buffers) { 1603 if (!Flush()) 1604 return kDecodeError; 1605 1606 available_va_surfaces_.clear(); 1607 return kAllocateNewSurfaces; 1608 } 1609 break; 1610 } 1611 1612 case H264NALU::kPPS: { 1613 if (state_ != kDecoding) 1614 break; 1615 1616 int pps_id; 1617 1618 if (!FinishPrevFrameIfPresent()) 1619 SET_ERROR_AND_RETURN(); 1620 1621 par_res = parser_.ParsePPS(&pps_id); 1622 if (par_res != H264Parser::kOk) 1623 SET_ERROR_AND_RETURN(); 1624 1625 if (!ProcessPPS(pps_id)) 1626 SET_ERROR_AND_RETURN(); 1627 break; 1628 } 1629 1630 default: 1631 DVLOG(4) << "Skipping NALU type: " << nalu.nal_unit_type;; 1632 break; 1633 } 1634 } 1635 } 1636 1637 size_t VaapiH264Decoder::GetRequiredNumOfPictures() { 1638 return dpb_.max_num_pics() + kPicsInPipeline; 1639 } 1640 1641 } // namespace content 1642
