1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include <algorithm> 6 #include <limits> 7 8 #include "base/bind.h" 9 #include "base/bind_helpers.h" 10 #include "base/stl_util.h" 11 #include "content/common/gpu/media/vaapi_h264_decoder.h" 12 13 namespace content { 14 15 // Decode surface, used for decoding and reference. input_id comes from client 16 // and is associated with the surface that was produced as the result 17 // of decoding a bitstream buffer with that id. 18 class VaapiH264Decoder::DecodeSurface { 19 public: 20 DecodeSurface(int poc, 21 int32 input_id, 22 const scoped_refptr<VASurface>& va_surface); 23 DecodeSurface(int poc, const scoped_refptr<DecodeSurface>& dec_surface); 24 ~DecodeSurface(); 25 26 int poc() { 27 return poc_; 28 } 29 30 scoped_refptr<VASurface> va_surface() { 31 return va_surface_; 32 } 33 34 int32 input_id() { 35 return input_id_; 36 } 37 38 private: 39 int poc_; 40 int32 input_id_; 41 scoped_refptr<VASurface> va_surface_; 42 }; 43 44 VaapiH264Decoder::DecodeSurface::DecodeSurface( 45 int poc, 46 int32 input_id, 47 const scoped_refptr<VASurface>& va_surface) 48 : poc_(poc), 49 input_id_(input_id), 50 va_surface_(va_surface) { 51 DCHECK(va_surface_.get()); 52 } 53 54 VaapiH264Decoder::DecodeSurface::~DecodeSurface() { 55 } 56 57 VaapiH264Decoder::VaapiH264Decoder( 58 VaapiWrapper* vaapi_wrapper, 59 const OutputPicCB& output_pic_cb, 60 const ReportErrorToUmaCB& report_error_to_uma_cb) 61 : max_pic_order_cnt_lsb_(0), 62 max_frame_num_(0), 63 max_pic_num_(0), 64 max_long_term_frame_idx_(0), 65 curr_sps_id_(-1), 66 curr_pps_id_(-1), 67 vaapi_wrapper_(vaapi_wrapper), 68 output_pic_cb_(output_pic_cb), 69 report_error_to_uma_cb_(report_error_to_uma_cb) { 70 Reset(); 71 state_ = kNeedStreamMetadata; 72 } 73 74 VaapiH264Decoder::~VaapiH264Decoder() { 75 } 76 77 void VaapiH264Decoder::Reset() { 78 curr_pic_.reset(); 79 80 curr_input_id_ = -1; 81 frame_num_ = 0; 82 prev_frame_num_ = -1; 83 prev_frame_num_offset_ = -1; 84 85 prev_ref_has_memmgmnt5_ = false; 86 prev_ref_top_field_order_cnt_ = -1; 87 prev_ref_pic_order_cnt_msb_ = -1; 88 prev_ref_pic_order_cnt_lsb_ = -1; 89 prev_ref_field_ = H264Picture::FIELD_NONE; 90 91 vaapi_wrapper_->DestroyPendingBuffers(); 92 93 ref_pic_list0_.clear(); 94 ref_pic_list1_.clear(); 95 96 for (DecSurfacesInUse::iterator it = decode_surfaces_in_use_.begin(); 97 it != decode_surfaces_in_use_.end(); ) { 98 int poc = it->second->poc(); 99 // Must be incremented before UnassignSurfaceFromPoC as this call 100 // invalidates |it|. 101 ++it; 102 UnassignSurfaceFromPoC(poc); 103 } 104 DCHECK(decode_surfaces_in_use_.empty()); 105 106 dpb_.Clear(); 107 parser_.Reset(); 108 last_output_poc_ = 0; 109 110 // If we are in kDecoding, we can resume without processing an SPS. 111 if (state_ == kDecoding) 112 state_ = kAfterReset; 113 } 114 115 void VaapiH264Decoder::ReuseSurface( 116 const scoped_refptr<VASurface>& va_surface) { 117 available_va_surfaces_.push_back(va_surface); 118 } 119 120 // Fill |va_pic| with default/neutral values. 121 static void InitVAPicture(VAPictureH264* va_pic) { 122 memset(va_pic, 0, sizeof(*va_pic)); 123 va_pic->picture_id = VA_INVALID_ID; 124 va_pic->flags = VA_PICTURE_H264_INVALID; 125 } 126 127 void VaapiH264Decoder::FillVAPicture(VAPictureH264 *va_pic, H264Picture* pic) { 128 DCHECK(pic); 129 130 DecodeSurface* dec_surface = DecodeSurfaceByPoC(pic->pic_order_cnt); 131 if (!dec_surface) { 132 // Cannot provide a ref picture, will corrupt output, but may be able 133 // to recover. 134 InitVAPicture(va_pic); 135 return; 136 } 137 138 va_pic->picture_id = dec_surface->va_surface()->id(); 139 va_pic->frame_idx = pic->frame_num; 140 va_pic->flags = 0; 141 142 switch (pic->field) { 143 case H264Picture::FIELD_NONE: 144 break; 145 case H264Picture::FIELD_TOP: 146 va_pic->flags |= VA_PICTURE_H264_TOP_FIELD; 147 break; 148 case H264Picture::FIELD_BOTTOM: 149 va_pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD; 150 break; 151 } 152 153 if (pic->ref) { 154 va_pic->flags |= pic->long_term ? VA_PICTURE_H264_LONG_TERM_REFERENCE 155 : VA_PICTURE_H264_SHORT_TERM_REFERENCE; 156 } 157 158 va_pic->TopFieldOrderCnt = pic->top_field_order_cnt; 159 va_pic->BottomFieldOrderCnt = pic->bottom_field_order_cnt; 160 } 161 162 int VaapiH264Decoder::FillVARefFramesFromDPB(VAPictureH264 *va_pics, 163 int num_pics) { 164 H264DPB::Pictures::reverse_iterator rit; 165 int i; 166 167 // Return reference frames in reverse order of insertion. 168 // Libva does not document this, but other implementations (e.g. mplayer) 169 // do it this way as well. 170 for (rit = dpb_.rbegin(), i = 0; rit != dpb_.rend() && i < num_pics; ++rit) { 171 if ((*rit)->ref) 172 FillVAPicture(&va_pics[i++], *rit); 173 } 174 175 return i; 176 } 177 178 VaapiH264Decoder::DecodeSurface* VaapiH264Decoder::DecodeSurfaceByPoC(int poc) { 179 DecSurfacesInUse::iterator iter = decode_surfaces_in_use_.find(poc); 180 if (iter == decode_surfaces_in_use_.end()) { 181 DVLOG(1) << "Could not find surface assigned to POC: " << poc; 182 return NULL; 183 } 184 185 return iter->second.get(); 186 } 187 188 bool VaapiH264Decoder::AssignSurfaceToPoC(int32 input_id, int poc) { 189 if (available_va_surfaces_.empty()) { 190 DVLOG(1) << "No VA Surfaces available"; 191 return false; 192 } 193 194 linked_ptr<DecodeSurface> dec_surface(new DecodeSurface( 195 poc, input_id, available_va_surfaces_.back())); 196 available_va_surfaces_.pop_back(); 197 198 DVLOG(4) << "POC " << poc 199 << " will use surface " << dec_surface->va_surface()->id(); 200 201 bool inserted = decode_surfaces_in_use_.insert( 202 std::make_pair(poc, dec_surface)).second; 203 DCHECK(inserted); 204 205 return true; 206 } 207 208 void VaapiH264Decoder::UnassignSurfaceFromPoC(int poc) { 209 DecSurfacesInUse::iterator it = decode_surfaces_in_use_.find(poc); 210 if (it == decode_surfaces_in_use_.end()) { 211 DVLOG(1) << "Asked to unassign an unassigned POC " << poc; 212 return; 213 } 214 215 DVLOG(4) << "POC " << poc << " no longer using VA surface " 216 << it->second->va_surface()->id(); 217 218 decode_surfaces_in_use_.erase(it); 219 } 220 221 bool VaapiH264Decoder::SendPPS() { 222 const H264PPS* pps = parser_.GetPPS(curr_pps_id_); 223 DCHECK(pps); 224 225 const H264SPS* sps = parser_.GetSPS(pps->seq_parameter_set_id); 226 DCHECK(sps); 227 228 DCHECK(curr_pic_.get()); 229 230 VAPictureParameterBufferH264 pic_param; 231 memset(&pic_param, 0, sizeof(VAPictureParameterBufferH264)); 232 233 #define FROM_SPS_TO_PP(a) pic_param.a = sps->a; 234 #define FROM_SPS_TO_PP2(a, b) pic_param.b = sps->a; 235 FROM_SPS_TO_PP2(pic_width_in_mbs_minus1, picture_width_in_mbs_minus1); 236 // This assumes non-interlaced video 237 FROM_SPS_TO_PP2(pic_height_in_map_units_minus1, 238 picture_height_in_mbs_minus1); 239 FROM_SPS_TO_PP(bit_depth_luma_minus8); 240 FROM_SPS_TO_PP(bit_depth_chroma_minus8); 241 #undef FROM_SPS_TO_PP 242 #undef FROM_SPS_TO_PP2 243 244 #define FROM_SPS_TO_PP_SF(a) pic_param.seq_fields.bits.a = sps->a; 245 #define FROM_SPS_TO_PP_SF2(a, b) pic_param.seq_fields.bits.b = sps->a; 246 FROM_SPS_TO_PP_SF(chroma_format_idc); 247 FROM_SPS_TO_PP_SF2(separate_colour_plane_flag, 248 residual_colour_transform_flag); 249 FROM_SPS_TO_PP_SF(gaps_in_frame_num_value_allowed_flag); 250 FROM_SPS_TO_PP_SF(frame_mbs_only_flag); 251 FROM_SPS_TO_PP_SF(mb_adaptive_frame_field_flag); 252 FROM_SPS_TO_PP_SF(direct_8x8_inference_flag); 253 pic_param.seq_fields.bits.MinLumaBiPredSize8x8 = (sps->level_idc >= 31); 254 FROM_SPS_TO_PP_SF(log2_max_frame_num_minus4); 255 FROM_SPS_TO_PP_SF(pic_order_cnt_type); 256 FROM_SPS_TO_PP_SF(log2_max_pic_order_cnt_lsb_minus4); 257 FROM_SPS_TO_PP_SF(delta_pic_order_always_zero_flag); 258 #undef FROM_SPS_TO_PP_SF 259 #undef FROM_SPS_TO_PP_SF2 260 261 #define FROM_PPS_TO_PP(a) pic_param.a = pps->a; 262 FROM_PPS_TO_PP(num_slice_groups_minus1); 263 pic_param.slice_group_map_type = 0; 264 pic_param.slice_group_change_rate_minus1 = 0; 265 FROM_PPS_TO_PP(pic_init_qp_minus26); 266 FROM_PPS_TO_PP(pic_init_qs_minus26); 267 FROM_PPS_TO_PP(chroma_qp_index_offset); 268 FROM_PPS_TO_PP(second_chroma_qp_index_offset); 269 #undef FROM_PPS_TO_PP 270 271 #define FROM_PPS_TO_PP_PF(a) pic_param.pic_fields.bits.a = pps->a; 272 #define FROM_PPS_TO_PP_PF2(a, b) pic_param.pic_fields.bits.b = pps->a; 273 FROM_PPS_TO_PP_PF(entropy_coding_mode_flag); 274 FROM_PPS_TO_PP_PF(weighted_pred_flag); 275 FROM_PPS_TO_PP_PF(weighted_bipred_idc); 276 FROM_PPS_TO_PP_PF(transform_8x8_mode_flag); 277 278 pic_param.pic_fields.bits.field_pic_flag = 0; 279 FROM_PPS_TO_PP_PF(constrained_intra_pred_flag); 280 FROM_PPS_TO_PP_PF2(bottom_field_pic_order_in_frame_present_flag, 281 pic_order_present_flag); 282 FROM_PPS_TO_PP_PF(deblocking_filter_control_present_flag); 283 FROM_PPS_TO_PP_PF(redundant_pic_cnt_present_flag); 284 pic_param.pic_fields.bits.reference_pic_flag = curr_pic_->ref; 285 #undef FROM_PPS_TO_PP_PF 286 #undef FROM_PPS_TO_PP_PF2 287 288 pic_param.frame_num = curr_pic_->frame_num; 289 290 InitVAPicture(&pic_param.CurrPic); 291 FillVAPicture(&pic_param.CurrPic, curr_pic_.get()); 292 293 // Init reference pictures' array. 294 for (int i = 0; i < 16; ++i) 295 InitVAPicture(&pic_param.ReferenceFrames[i]); 296 297 // And fill it with picture info from DPB. 298 FillVARefFramesFromDPB(pic_param.ReferenceFrames, 299 arraysize(pic_param.ReferenceFrames)); 300 301 pic_param.num_ref_frames = sps->max_num_ref_frames; 302 303 return vaapi_wrapper_->SubmitBuffer(VAPictureParameterBufferType, 304 sizeof(VAPictureParameterBufferH264), 305 &pic_param); 306 } 307 308 bool VaapiH264Decoder::SendIQMatrix() { 309 const H264PPS* pps = parser_.GetPPS(curr_pps_id_); 310 DCHECK(pps); 311 312 VAIQMatrixBufferH264 iq_matrix_buf; 313 memset(&iq_matrix_buf, 0, sizeof(VAIQMatrixBufferH264)); 314 315 if (pps->pic_scaling_matrix_present_flag) { 316 for (int i = 0; i < 6; ++i) { 317 for (int j = 0; j < 16; ++j) 318 iq_matrix_buf.ScalingList4x4[i][j] = pps->scaling_list4x4[i][j]; 319 } 320 321 for (int i = 0; i < 2; ++i) { 322 for (int j = 0; j < 64; ++j) 323 iq_matrix_buf.ScalingList8x8[i][j] = pps->scaling_list8x8[i][j]; 324 } 325 } else { 326 const H264SPS* sps = parser_.GetSPS(pps->seq_parameter_set_id); 327 DCHECK(sps); 328 for (int i = 0; i < 6; ++i) { 329 for (int j = 0; j < 16; ++j) 330 iq_matrix_buf.ScalingList4x4[i][j] = sps->scaling_list4x4[i][j]; 331 } 332 333 for (int i = 0; i < 2; ++i) { 334 for (int j = 0; j < 64; ++j) 335 iq_matrix_buf.ScalingList8x8[i][j] = sps->scaling_list8x8[i][j]; 336 } 337 } 338 339 return vaapi_wrapper_->SubmitBuffer(VAIQMatrixBufferType, 340 sizeof(VAIQMatrixBufferH264), 341 &iq_matrix_buf); 342 } 343 344 bool VaapiH264Decoder::SendVASliceParam(H264SliceHeader* slice_hdr) { 345 const H264PPS* pps = parser_.GetPPS(slice_hdr->pic_parameter_set_id); 346 DCHECK(pps); 347 348 const H264SPS* sps = parser_.GetSPS(pps->seq_parameter_set_id); 349 DCHECK(sps); 350 351 VASliceParameterBufferH264 slice_param; 352 memset(&slice_param, 0, sizeof(VASliceParameterBufferH264)); 353 354 slice_param.slice_data_size = slice_hdr->nalu_size; 355 slice_param.slice_data_offset = 0; 356 slice_param.slice_data_flag = VA_SLICE_DATA_FLAG_ALL; 357 slice_param.slice_data_bit_offset = slice_hdr->header_bit_size; 358 359 #define SHDRToSP(a) slice_param.a = slice_hdr->a; 360 SHDRToSP(first_mb_in_slice); 361 slice_param.slice_type = slice_hdr->slice_type % 5; 362 SHDRToSP(direct_spatial_mv_pred_flag); 363 364 // TODO posciak: make sure parser sets those even when override flags 365 // in slice header is off. 366 SHDRToSP(num_ref_idx_l0_active_minus1); 367 SHDRToSP(num_ref_idx_l1_active_minus1); 368 SHDRToSP(cabac_init_idc); 369 SHDRToSP(slice_qp_delta); 370 SHDRToSP(disable_deblocking_filter_idc); 371 SHDRToSP(slice_alpha_c0_offset_div2); 372 SHDRToSP(slice_beta_offset_div2); 373 374 if (((slice_hdr->IsPSlice() || slice_hdr->IsSPSlice()) && 375 pps->weighted_pred_flag) || 376 (slice_hdr->IsBSlice() && pps->weighted_bipred_idc == 1)) { 377 SHDRToSP(luma_log2_weight_denom); 378 SHDRToSP(chroma_log2_weight_denom); 379 380 SHDRToSP(luma_weight_l0_flag); 381 SHDRToSP(luma_weight_l1_flag); 382 383 SHDRToSP(chroma_weight_l0_flag); 384 SHDRToSP(chroma_weight_l1_flag); 385 386 for (int i = 0; i <= slice_param.num_ref_idx_l0_active_minus1; ++i) { 387 slice_param.luma_weight_l0[i] = 388 slice_hdr->pred_weight_table_l0.luma_weight[i]; 389 slice_param.luma_offset_l0[i] = 390 slice_hdr->pred_weight_table_l0.luma_offset[i]; 391 392 for (int j = 0; j < 2; ++j) { 393 slice_param.chroma_weight_l0[i][j] = 394 slice_hdr->pred_weight_table_l0.chroma_weight[i][j]; 395 slice_param.chroma_offset_l0[i][j] = 396 slice_hdr->pred_weight_table_l0.chroma_offset[i][j]; 397 } 398 } 399 400 if (slice_hdr->IsBSlice()) { 401 for (int i = 0; i <= slice_param.num_ref_idx_l1_active_minus1; ++i) { 402 slice_param.luma_weight_l1[i] = 403 slice_hdr->pred_weight_table_l1.luma_weight[i]; 404 slice_param.luma_offset_l1[i] = 405 slice_hdr->pred_weight_table_l1.luma_offset[i]; 406 407 for (int j = 0; j < 2; ++j) { 408 slice_param.chroma_weight_l1[i][j] = 409 slice_hdr->pred_weight_table_l1.chroma_weight[i][j]; 410 slice_param.chroma_offset_l1[i][j] = 411 slice_hdr->pred_weight_table_l1.chroma_offset[i][j]; 412 } 413 } 414 } 415 } 416 417 for (int i = 0; i < 32; ++i) { 418 InitVAPicture(&slice_param.RefPicList0[i]); 419 InitVAPicture(&slice_param.RefPicList1[i]); 420 } 421 422 int i; 423 H264Picture::PtrVector::iterator it; 424 for (it = ref_pic_list0_.begin(), i = 0; it != ref_pic_list0_.end() && *it; 425 ++it, ++i) 426 FillVAPicture(&slice_param.RefPicList0[i], *it); 427 for (it = ref_pic_list1_.begin(), i = 0; it != ref_pic_list1_.end() && *it; 428 ++it, ++i) 429 FillVAPicture(&slice_param.RefPicList1[i], *it); 430 431 return vaapi_wrapper_->SubmitBuffer(VASliceParameterBufferType, 432 sizeof(VASliceParameterBufferH264), 433 &slice_param); 434 } 435 436 bool VaapiH264Decoder::SendSliceData(const uint8* ptr, size_t size) { 437 // Can't help it, blame libva... 438 void* non_const_ptr = const_cast<uint8*>(ptr); 439 return vaapi_wrapper_->SubmitBuffer(VASliceDataBufferType, size, 440 non_const_ptr); 441 } 442 443 bool VaapiH264Decoder::QueueSlice(H264SliceHeader* slice_hdr) { 444 DCHECK(curr_pic_.get()); 445 446 if (!SendVASliceParam(slice_hdr)) 447 return false; 448 449 if (!SendSliceData(slice_hdr->nalu_data, slice_hdr->nalu_size)) 450 return false; 451 452 return true; 453 } 454 455 // TODO(posciak) start using vaMapBuffer instead of vaCreateBuffer wherever 456 // possible. 457 bool VaapiH264Decoder::DecodePicture() { 458 DCHECK(curr_pic_.get()); 459 460 DVLOG(4) << "Decoding POC " << curr_pic_->pic_order_cnt; 461 DecodeSurface* dec_surface = DecodeSurfaceByPoC(curr_pic_->pic_order_cnt); 462 if (!dec_surface) { 463 DVLOG(1) << "Asked to decode an invalid POC " << curr_pic_->pic_order_cnt; 464 return false; 465 } 466 467 if (!vaapi_wrapper_->DecodeAndDestroyPendingBuffers( 468 dec_surface->va_surface()->id())) { 469 DVLOG(1) << "Failed decoding picture"; 470 return false; 471 } 472 473 return true; 474 } 475 476 477 bool VaapiH264Decoder::InitCurrPicture(H264SliceHeader* slice_hdr) { 478 DCHECK(curr_pic_.get()); 479 480 memset(curr_pic_.get(), 0, sizeof(H264Picture)); 481 482 curr_pic_->idr = slice_hdr->idr_pic_flag; 483 484 if (slice_hdr->field_pic_flag) { 485 curr_pic_->field = slice_hdr->bottom_field_flag ? H264Picture::FIELD_BOTTOM 486 : H264Picture::FIELD_TOP; 487 } else { 488 curr_pic_->field = H264Picture::FIELD_NONE; 489 } 490 491 curr_pic_->ref = slice_hdr->nal_ref_idc != 0; 492 // This assumes non-interlaced stream. 493 curr_pic_->frame_num = curr_pic_->pic_num = slice_hdr->frame_num; 494 495 if (!CalculatePicOrderCounts(slice_hdr)) 496 return false; 497 498 // Try to get an empty surface to decode this picture to. 499 if (!AssignSurfaceToPoC(curr_input_id_, curr_pic_->pic_order_cnt)) { 500 DVLOG(1) << "Failed getting a free surface for a picture"; 501 return false; 502 } 503 504 curr_pic_->long_term_reference_flag = slice_hdr->long_term_reference_flag; 505 curr_pic_->adaptive_ref_pic_marking_mode_flag = 506 slice_hdr->adaptive_ref_pic_marking_mode_flag; 507 508 // If the slice header indicates we will have to perform reference marking 509 // process after this picture is decoded, store required data for that 510 // purpose. 511 if (slice_hdr->adaptive_ref_pic_marking_mode_flag) { 512 COMPILE_ASSERT(sizeof(curr_pic_->ref_pic_marking) == 513 sizeof(slice_hdr->ref_pic_marking), 514 ref_pic_marking_array_sizes_do_not_match); 515 memcpy(curr_pic_->ref_pic_marking, slice_hdr->ref_pic_marking, 516 sizeof(curr_pic_->ref_pic_marking)); 517 } 518 519 return true; 520 } 521 522 bool VaapiH264Decoder::CalculatePicOrderCounts(H264SliceHeader* slice_hdr) { 523 DCHECK_NE(curr_sps_id_, -1); 524 const H264SPS* sps = parser_.GetSPS(curr_sps_id_); 525 526 int pic_order_cnt_lsb = slice_hdr->pic_order_cnt_lsb; 527 curr_pic_->pic_order_cnt_lsb = pic_order_cnt_lsb; 528 529 switch (sps->pic_order_cnt_type) { 530 case 0: 531 // See spec 8.2.1.1. 532 int prev_pic_order_cnt_msb, prev_pic_order_cnt_lsb; 533 if (slice_hdr->idr_pic_flag) { 534 prev_pic_order_cnt_msb = prev_pic_order_cnt_lsb = 0; 535 } else { 536 if (prev_ref_has_memmgmnt5_) { 537 if (prev_ref_field_ != H264Picture::FIELD_BOTTOM) { 538 prev_pic_order_cnt_msb = 0; 539 prev_pic_order_cnt_lsb = prev_ref_top_field_order_cnt_; 540 } else { 541 prev_pic_order_cnt_msb = 0; 542 prev_pic_order_cnt_lsb = 0; 543 } 544 } else { 545 prev_pic_order_cnt_msb = prev_ref_pic_order_cnt_msb_; 546 prev_pic_order_cnt_lsb = prev_ref_pic_order_cnt_lsb_; 547 } 548 } 549 550 DCHECK_NE(max_pic_order_cnt_lsb_, 0); 551 if ((pic_order_cnt_lsb < prev_pic_order_cnt_lsb) && 552 (prev_pic_order_cnt_lsb - pic_order_cnt_lsb >= 553 max_pic_order_cnt_lsb_ / 2)) { 554 curr_pic_->pic_order_cnt_msb = prev_pic_order_cnt_msb + 555 max_pic_order_cnt_lsb_; 556 } else if ((pic_order_cnt_lsb > prev_pic_order_cnt_lsb) && 557 (pic_order_cnt_lsb - prev_pic_order_cnt_lsb > 558 max_pic_order_cnt_lsb_ / 2)) { 559 curr_pic_->pic_order_cnt_msb = prev_pic_order_cnt_msb - 560 max_pic_order_cnt_lsb_; 561 } else { 562 curr_pic_->pic_order_cnt_msb = prev_pic_order_cnt_msb; 563 } 564 565 if (curr_pic_->field != H264Picture::FIELD_BOTTOM) { 566 curr_pic_->top_field_order_cnt = curr_pic_->pic_order_cnt_msb + 567 pic_order_cnt_lsb; 568 } 569 570 if (curr_pic_->field != H264Picture::FIELD_TOP) { 571 // TODO posciak: perhaps replace with pic->field? 572 if (!slice_hdr->field_pic_flag) { 573 curr_pic_->bottom_field_order_cnt = curr_pic_->top_field_order_cnt + 574 slice_hdr->delta_pic_order_cnt_bottom; 575 } else { 576 curr_pic_->bottom_field_order_cnt = curr_pic_->pic_order_cnt_msb + 577 pic_order_cnt_lsb; 578 } 579 } 580 break; 581 582 case 1: { 583 // See spec 8.2.1.2. 584 if (prev_has_memmgmnt5_) 585 prev_frame_num_offset_ = 0; 586 587 if (slice_hdr->idr_pic_flag) 588 curr_pic_->frame_num_offset = 0; 589 else if (prev_frame_num_ > slice_hdr->frame_num) 590 curr_pic_->frame_num_offset = prev_frame_num_offset_ + max_frame_num_; 591 else 592 curr_pic_->frame_num_offset = prev_frame_num_offset_; 593 594 int abs_frame_num = 0; 595 if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0) 596 abs_frame_num = curr_pic_->frame_num_offset + slice_hdr->frame_num; 597 else 598 abs_frame_num = 0; 599 600 if (slice_hdr->nal_ref_idc == 0 && abs_frame_num > 0) 601 --abs_frame_num; 602 603 int expected_pic_order_cnt = 0; 604 if (abs_frame_num > 0) { 605 if (sps->num_ref_frames_in_pic_order_cnt_cycle == 0) { 606 DVLOG(1) << "Invalid num_ref_frames_in_pic_order_cnt_cycle " 607 << "in stream"; 608 return false; 609 } 610 611 int pic_order_cnt_cycle_cnt = (abs_frame_num - 1) / 612 sps->num_ref_frames_in_pic_order_cnt_cycle; 613 int frame_num_in_pic_order_cnt_cycle = (abs_frame_num - 1) % 614 sps->num_ref_frames_in_pic_order_cnt_cycle; 615 616 expected_pic_order_cnt = pic_order_cnt_cycle_cnt * 617 sps->expected_delta_per_pic_order_cnt_cycle; 618 // frame_num_in_pic_order_cnt_cycle is verified < 255 in parser 619 for (int i = 0; i <= frame_num_in_pic_order_cnt_cycle; ++i) 620 expected_pic_order_cnt += sps->offset_for_ref_frame[i]; 621 } 622 623 if (!slice_hdr->nal_ref_idc) 624 expected_pic_order_cnt += sps->offset_for_non_ref_pic; 625 626 if (!slice_hdr->field_pic_flag) { 627 curr_pic_->top_field_order_cnt = expected_pic_order_cnt + 628 slice_hdr->delta_pic_order_cnt[0]; 629 curr_pic_->bottom_field_order_cnt = curr_pic_->top_field_order_cnt + 630 sps->offset_for_top_to_bottom_field + 631 slice_hdr->delta_pic_order_cnt[1]; 632 } else if (!slice_hdr->bottom_field_flag) { 633 curr_pic_->top_field_order_cnt = expected_pic_order_cnt + 634 slice_hdr->delta_pic_order_cnt[0]; 635 } else { 636 curr_pic_->bottom_field_order_cnt = expected_pic_order_cnt + 637 sps->offset_for_top_to_bottom_field + 638 slice_hdr->delta_pic_order_cnt[0]; 639 } 640 break; 641 } 642 643 case 2: 644 // See spec 8.2.1.3. 645 if (prev_has_memmgmnt5_) 646 prev_frame_num_offset_ = 0; 647 648 if (slice_hdr->idr_pic_flag) 649 curr_pic_->frame_num_offset = 0; 650 else if (prev_frame_num_ > slice_hdr->frame_num) 651 curr_pic_->frame_num_offset = prev_frame_num_offset_ + max_frame_num_; 652 else 653 curr_pic_->frame_num_offset = prev_frame_num_offset_; 654 655 int temp_pic_order_cnt; 656 if (slice_hdr->idr_pic_flag) { 657 temp_pic_order_cnt = 0; 658 } else if (!slice_hdr->nal_ref_idc) { 659 temp_pic_order_cnt = 660 2 * (curr_pic_->frame_num_offset + slice_hdr->frame_num) - 1; 661 } else { 662 temp_pic_order_cnt = 2 * (curr_pic_->frame_num_offset + 663 slice_hdr->frame_num); 664 } 665 666 if (!slice_hdr->field_pic_flag) { 667 curr_pic_->top_field_order_cnt = temp_pic_order_cnt; 668 curr_pic_->bottom_field_order_cnt = temp_pic_order_cnt; 669 } else if (slice_hdr->bottom_field_flag) { 670 curr_pic_->bottom_field_order_cnt = temp_pic_order_cnt; 671 } else { 672 curr_pic_->top_field_order_cnt = temp_pic_order_cnt; 673 } 674 break; 675 676 default: 677 DVLOG(1) << "Invalid pic_order_cnt_type: " << sps->pic_order_cnt_type; 678 return false; 679 } 680 681 switch (curr_pic_->field) { 682 case H264Picture::FIELD_NONE: 683 curr_pic_->pic_order_cnt = std::min(curr_pic_->top_field_order_cnt, 684 curr_pic_->bottom_field_order_cnt); 685 break; 686 case H264Picture::FIELD_TOP: 687 curr_pic_->pic_order_cnt = curr_pic_->top_field_order_cnt; 688 break; 689 case H264Picture::FIELD_BOTTOM: 690 curr_pic_->pic_order_cnt = curr_pic_->bottom_field_order_cnt; 691 break; 692 } 693 694 return true; 695 } 696 697 void VaapiH264Decoder::UpdatePicNums() { 698 for (H264DPB::Pictures::iterator it = dpb_.begin(); it != dpb_.end(); ++it) { 699 H264Picture* pic = *it; 700 DCHECK(pic); 701 if (!pic->ref) 702 continue; 703 704 // Below assumes non-interlaced stream. 705 DCHECK_EQ(pic->field, H264Picture::FIELD_NONE); 706 if (pic->long_term) { 707 pic->long_term_pic_num = pic->long_term_frame_idx; 708 } else { 709 if (pic->frame_num > frame_num_) 710 pic->frame_num_wrap = pic->frame_num - max_frame_num_; 711 else 712 pic->frame_num_wrap = pic->frame_num; 713 714 pic->pic_num = pic->frame_num_wrap; 715 } 716 } 717 } 718 719 struct PicNumDescCompare { 720 bool operator()(const H264Picture* a, const H264Picture* b) const { 721 return a->pic_num > b->pic_num; 722 } 723 }; 724 725 struct LongTermPicNumAscCompare { 726 bool operator()(const H264Picture* a, const H264Picture* b) const { 727 return a->long_term_pic_num < b->long_term_pic_num; 728 } 729 }; 730 731 void VaapiH264Decoder::ConstructReferencePicListsP(H264SliceHeader* slice_hdr) { 732 // RefPicList0 (8.2.4.2.1) [[1] [2]], where: 733 // [1] shortterm ref pics sorted by descending pic_num, 734 // [2] longterm ref pics by ascending long_term_pic_num. 735 DCHECK(ref_pic_list0_.empty() && ref_pic_list1_.empty()); 736 // First get the short ref pics... 737 dpb_.GetShortTermRefPicsAppending(ref_pic_list0_); 738 size_t num_short_refs = ref_pic_list0_.size(); 739 740 // and sort them to get [1]. 741 std::sort(ref_pic_list0_.begin(), ref_pic_list0_.end(), PicNumDescCompare()); 742 743 // Now get long term pics and sort them by long_term_pic_num to get [2]. 744 dpb_.GetLongTermRefPicsAppending(ref_pic_list0_); 745 std::sort(ref_pic_list0_.begin() + num_short_refs, ref_pic_list0_.end(), 746 LongTermPicNumAscCompare()); 747 748 // Cut off if we have more than requested in slice header. 749 ref_pic_list0_.resize(slice_hdr->num_ref_idx_l0_active_minus1 + 1); 750 } 751 752 struct POCAscCompare { 753 bool operator()(const H264Picture* a, const H264Picture* b) const { 754 return a->pic_order_cnt < b->pic_order_cnt; 755 } 756 }; 757 758 struct POCDescCompare { 759 bool operator()(const H264Picture* a, const H264Picture* b) const { 760 return a->pic_order_cnt > b->pic_order_cnt; 761 } 762 }; 763 764 void VaapiH264Decoder::ConstructReferencePicListsB(H264SliceHeader* slice_hdr) { 765 // RefPicList0 (8.2.4.2.3) [[1] [2] [3]], where: 766 // [1] shortterm ref pics with POC < curr_pic's POC sorted by descending POC, 767 // [2] shortterm ref pics with POC > curr_pic's POC by ascending POC, 768 // [3] longterm ref pics by ascending long_term_pic_num. 769 DCHECK(ref_pic_list0_.empty() && ref_pic_list1_.empty()); 770 dpb_.GetShortTermRefPicsAppending(ref_pic_list0_); 771 size_t num_short_refs = ref_pic_list0_.size(); 772 773 // First sort ascending, this will put [1] in right place and finish [2]. 774 std::sort(ref_pic_list0_.begin(), ref_pic_list0_.end(), POCAscCompare()); 775 776 // Find first with POC > curr_pic's POC to get first element in [2]... 777 H264Picture::PtrVector::iterator iter; 778 iter = std::upper_bound(ref_pic_list0_.begin(), ref_pic_list0_.end(), 779 curr_pic_.get(), POCAscCompare()); 780 781 // and sort [1] descending, thus finishing sequence [1] [2]. 782 std::sort(ref_pic_list0_.begin(), iter, POCDescCompare()); 783 784 // Now add [3] and sort by ascending long_term_pic_num. 785 dpb_.GetLongTermRefPicsAppending(ref_pic_list0_); 786 std::sort(ref_pic_list0_.begin() + num_short_refs, ref_pic_list0_.end(), 787 LongTermPicNumAscCompare()); 788 789 // RefPicList1 (8.2.4.2.4) [[1] [2] [3]], where: 790 // [1] shortterm ref pics with POC > curr_pic's POC sorted by ascending POC, 791 // [2] shortterm ref pics with POC < curr_pic's POC by descending POC, 792 // [3] longterm ref pics by ascending long_term_pic_num. 793 794 dpb_.GetShortTermRefPicsAppending(ref_pic_list1_); 795 num_short_refs = ref_pic_list1_.size(); 796 797 // First sort by descending POC. 798 std::sort(ref_pic_list1_.begin(), ref_pic_list1_.end(), POCDescCompare()); 799 800 // Find first with POC < curr_pic's POC to get first element in [2]... 801 iter = std::upper_bound(ref_pic_list1_.begin(), ref_pic_list1_.end(), 802 curr_pic_.get(), POCDescCompare()); 803 804 // and sort [1] ascending. 805 std::sort(ref_pic_list1_.begin(), iter, POCAscCompare()); 806 807 // Now add [3] and sort by ascending long_term_pic_num 808 dpb_.GetShortTermRefPicsAppending(ref_pic_list1_); 809 std::sort(ref_pic_list1_.begin() + num_short_refs, ref_pic_list1_.end(), 810 LongTermPicNumAscCompare()); 811 812 // If lists identical, swap first two entries in RefPicList1 (spec 8.2.4.2.3) 813 if (ref_pic_list1_.size() > 1 && 814 std::equal(ref_pic_list0_.begin(), ref_pic_list0_.end(), 815 ref_pic_list1_.begin())) 816 std::swap(ref_pic_list1_[0], ref_pic_list1_[1]); 817 818 // Per 8.2.4.2 it's possible for num_ref_idx_lX_active_minus1 to indicate 819 // there should be more ref pics on list than we constructed. 820 // Those superfluous ones should be treated as non-reference. 821 ref_pic_list0_.resize(slice_hdr->num_ref_idx_l0_active_minus1 + 1); 822 ref_pic_list1_.resize(slice_hdr->num_ref_idx_l1_active_minus1 + 1); 823 } 824 825 // See 8.2.4 826 int VaapiH264Decoder::PicNumF(H264Picture *pic) { 827 if (!pic) 828 return -1; 829 830 if (!pic->long_term) 831 return pic->pic_num; 832 else 833 return max_pic_num_; 834 } 835 836 // See 8.2.4 837 int VaapiH264Decoder::LongTermPicNumF(H264Picture *pic) { 838 if (pic->ref && pic->long_term) 839 return pic->long_term_pic_num; 840 else 841 return 2 * (max_long_term_frame_idx_ + 1); 842 } 843 844 // Shift elements on the |v| starting from |from| to |to|, inclusive, 845 // one position to the right and insert pic at |from|. 846 static void ShiftRightAndInsert(H264Picture::PtrVector *v, 847 int from, 848 int to, 849 H264Picture* pic) { 850 // Security checks, do not disable in Debug mode. 851 CHECK(from <= to); 852 CHECK(to <= std::numeric_limits<int>::max() - 2); 853 // Additional checks. Debug mode ok. 854 DCHECK(v); 855 DCHECK(pic); 856 DCHECK((to + 1 == static_cast<int>(v->size())) || 857 (to + 2 == static_cast<int>(v->size()))); 858 859 v->resize(to + 2); 860 861 for (int i = to + 1; i > from; --i) 862 (*v)[i] = (*v)[i - 1]; 863 864 (*v)[from] = pic; 865 } 866 867 bool VaapiH264Decoder::ModifyReferencePicList(H264SliceHeader *slice_hdr, 868 int list) { 869 int num_ref_idx_lX_active_minus1; 870 H264Picture::PtrVector* ref_pic_listx; 871 H264ModificationOfPicNum* list_mod; 872 873 // This can process either ref_pic_list0 or ref_pic_list1, depending on 874 // the list argument. Set up pointers to proper list to be processed here. 875 if (list == 0) { 876 if (!slice_hdr->ref_pic_list_modification_flag_l0) 877 return true; 878 879 list_mod = slice_hdr->ref_list_l0_modifications; 880 num_ref_idx_lX_active_minus1 = ref_pic_list0_.size() - 1; 881 882 ref_pic_listx = &ref_pic_list0_; 883 } else { 884 if (!slice_hdr->ref_pic_list_modification_flag_l1) 885 return true; 886 887 list_mod = slice_hdr->ref_list_l1_modifications; 888 num_ref_idx_lX_active_minus1 = ref_pic_list1_.size() - 1; 889 890 ref_pic_listx = &ref_pic_list1_; 891 } 892 893 DCHECK_GE(num_ref_idx_lX_active_minus1, 0); 894 895 // Spec 8.2.4.3: 896 // Reorder pictures on the list in a way specified in the stream. 897 int pic_num_lx_pred = curr_pic_->pic_num; 898 int ref_idx_lx = 0; 899 int pic_num_lx_no_wrap; 900 int pic_num_lx; 901 bool done = false; 902 H264Picture* pic; 903 for (int i = 0; i < H264SliceHeader::kRefListModSize && !done; ++i) { 904 switch (list_mod->modification_of_pic_nums_idc) { 905 case 0: 906 case 1: 907 // Modify short reference picture position. 908 if (list_mod->modification_of_pic_nums_idc == 0) { 909 // Subtract given value from predicted PicNum. 910 pic_num_lx_no_wrap = pic_num_lx_pred - 911 (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1); 912 // Wrap around max_pic_num_ if it becomes < 0 as result 913 // of subtraction. 914 if (pic_num_lx_no_wrap < 0) 915 pic_num_lx_no_wrap += max_pic_num_; 916 } else { 917 // Add given value to predicted PicNum. 918 pic_num_lx_no_wrap = pic_num_lx_pred + 919 (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1); 920 // Wrap around max_pic_num_ if it becomes >= max_pic_num_ as result 921 // of the addition. 922 if (pic_num_lx_no_wrap >= max_pic_num_) 923 pic_num_lx_no_wrap -= max_pic_num_; 924 } 925 926 // For use in next iteration. 927 pic_num_lx_pred = pic_num_lx_no_wrap; 928 929 if (pic_num_lx_no_wrap > curr_pic_->pic_num) 930 pic_num_lx = pic_num_lx_no_wrap - max_pic_num_; 931 else 932 pic_num_lx = pic_num_lx_no_wrap; 933 934 DCHECK_LT(num_ref_idx_lX_active_minus1 + 1, 935 H264SliceHeader::kRefListModSize); 936 pic = dpb_.GetShortRefPicByPicNum(pic_num_lx); 937 if (!pic) { 938 DVLOG(1) << "Malformed stream, no pic num " << pic_num_lx; 939 return false; 940 } 941 ShiftRightAndInsert(ref_pic_listx, ref_idx_lx, 942 num_ref_idx_lX_active_minus1, pic); 943 ref_idx_lx++; 944 945 for (int src = ref_idx_lx, dst = ref_idx_lx; 946 src <= num_ref_idx_lX_active_minus1 + 1; ++src) { 947 if (PicNumF((*ref_pic_listx)[src]) != pic_num_lx) 948 (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src]; 949 } 950 break; 951 952 case 2: 953 // Modify long term reference picture position. 954 DCHECK_LT(num_ref_idx_lX_active_minus1 + 1, 955 H264SliceHeader::kRefListModSize); 956 pic = dpb_.GetLongRefPicByLongTermPicNum(list_mod->long_term_pic_num); 957 if (!pic) { 958 DVLOG(1) << "Malformed stream, no pic num " 959 << list_mod->long_term_pic_num; 960 return false; 961 } 962 ShiftRightAndInsert(ref_pic_listx, ref_idx_lx, 963 num_ref_idx_lX_active_minus1, pic); 964 ref_idx_lx++; 965 966 for (int src = ref_idx_lx, dst = ref_idx_lx; 967 src <= num_ref_idx_lX_active_minus1 + 1; ++src) { 968 if (LongTermPicNumF((*ref_pic_listx)[src]) 969 != static_cast<int>(list_mod->long_term_pic_num)) 970 (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src]; 971 } 972 break; 973 974 case 3: 975 // End of modification list. 976 done = true; 977 break; 978 979 default: 980 // May be recoverable. 981 DVLOG(1) << "Invalid modification_of_pic_nums_idc=" 982 << list_mod->modification_of_pic_nums_idc 983 << " in position " << i; 984 break; 985 } 986 987 ++list_mod; 988 } 989 990 // Per NOTE 2 in 8.2.4.3.2, the ref_pic_listx size in the above loop is 991 // temporarily made one element longer than the required final list. 992 // Resize the list back to its required size. 993 ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1); 994 995 return true; 996 } 997 998 bool VaapiH264Decoder::OutputPic(H264Picture* pic) { 999 DCHECK(!pic->outputted); 1000 pic->outputted = true; 1001 last_output_poc_ = pic->pic_order_cnt; 1002 1003 DecodeSurface* dec_surface = DecodeSurfaceByPoC(pic->pic_order_cnt); 1004 if (!dec_surface) 1005 return false; 1006 1007 DCHECK_GE(dec_surface->input_id(), 0); 1008 DVLOG(4) << "Posting output task for POC: " << pic->pic_order_cnt 1009 << " input_id: " << dec_surface->input_id(); 1010 output_pic_cb_.Run(dec_surface->input_id(), dec_surface->va_surface()); 1011 1012 return true; 1013 } 1014 1015 void VaapiH264Decoder::ClearDPB() { 1016 // Clear DPB contents, marking the pictures as unused first. 1017 for (H264DPB::Pictures::iterator it = dpb_.begin(); it != dpb_.end(); ++it) 1018 UnassignSurfaceFromPoC((*it)->pic_order_cnt); 1019 1020 dpb_.Clear(); 1021 last_output_poc_ = 0; 1022 } 1023 1024 bool VaapiH264Decoder::OutputAllRemainingPics() { 1025 // Output all pictures that are waiting to be outputted. 1026 FinishPrevFrameIfPresent(); 1027 H264Picture::PtrVector to_output; 1028 dpb_.GetNotOutputtedPicsAppending(to_output); 1029 // Sort them by ascending POC to output in order. 1030 std::sort(to_output.begin(), to_output.end(), POCAscCompare()); 1031 1032 H264Picture::PtrVector::iterator it; 1033 for (it = to_output.begin(); it != to_output.end(); ++it) { 1034 if (!OutputPic(*it)) { 1035 DVLOG(1) << "Failed to output pic POC: " << (*it)->pic_order_cnt; 1036 return false; 1037 } 1038 } 1039 1040 return true; 1041 } 1042 1043 bool VaapiH264Decoder::Flush() { 1044 DVLOG(2) << "Decoder flush"; 1045 1046 if (!OutputAllRemainingPics()) 1047 return false; 1048 1049 ClearDPB(); 1050 1051 DCHECK(decode_surfaces_in_use_.empty()); 1052 return true; 1053 } 1054 1055 bool VaapiH264Decoder::StartNewFrame(H264SliceHeader* slice_hdr) { 1056 // TODO posciak: add handling of max_num_ref_frames per spec. 1057 1058 // If the new frame is an IDR, output what's left to output and clear DPB 1059 if (slice_hdr->idr_pic_flag) { 1060 // (unless we are explicitly instructed not to do so). 1061 if (!slice_hdr->no_output_of_prior_pics_flag) { 1062 // Output DPB contents. 1063 if (!Flush()) 1064 return false; 1065 } 1066 dpb_.Clear(); 1067 last_output_poc_ = 0; 1068 } 1069 1070 // curr_pic_ should have either been added to DPB or discarded when finishing 1071 // the last frame. DPB is responsible for releasing that memory once it's 1072 // not needed anymore. 1073 DCHECK(!curr_pic_.get()); 1074 curr_pic_.reset(new H264Picture); 1075 CHECK(curr_pic_.get()); 1076 1077 if (!InitCurrPicture(slice_hdr)) 1078 return false; 1079 1080 DCHECK_GT(max_frame_num_, 0); 1081 1082 UpdatePicNums(); 1083 1084 // Prepare reference picture lists if required (B and S/SP slices). 1085 ref_pic_list0_.clear(); 1086 ref_pic_list1_.clear(); 1087 if (slice_hdr->IsPSlice() || slice_hdr->IsSPSlice()) { 1088 ConstructReferencePicListsP(slice_hdr); 1089 if (!ModifyReferencePicList(slice_hdr, 0)) 1090 return false; 1091 } else if (slice_hdr->IsBSlice()) { 1092 ConstructReferencePicListsB(slice_hdr); 1093 if (!ModifyReferencePicList(slice_hdr, 0)) 1094 return false; 1095 if (!ModifyReferencePicList(slice_hdr, 1)) 1096 return false; 1097 } 1098 1099 // Send parameter buffers before each new picture, before the first slice. 1100 if (!SendPPS()) 1101 return false; 1102 1103 if (!SendIQMatrix()) 1104 return false; 1105 1106 if (!QueueSlice(slice_hdr)) 1107 return false; 1108 1109 return true; 1110 } 1111 1112 bool VaapiH264Decoder::HandleMemoryManagementOps() { 1113 // 8.2.5.4 1114 for (unsigned int i = 0; i < arraysize(curr_pic_->ref_pic_marking); ++i) { 1115 // Code below does not support interlaced stream (per-field pictures). 1116 H264DecRefPicMarking* ref_pic_marking = &curr_pic_->ref_pic_marking[i]; 1117 H264Picture* to_mark; 1118 int pic_num_x; 1119 1120 switch (ref_pic_marking->memory_mgmnt_control_operation) { 1121 case 0: 1122 // Normal end of operations' specification. 1123 return true; 1124 1125 case 1: 1126 // Mark a short term reference picture as unused so it can be removed 1127 // if outputted. 1128 pic_num_x = curr_pic_->pic_num - 1129 (ref_pic_marking->difference_of_pic_nums_minus1 + 1); 1130 to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x); 1131 if (to_mark) { 1132 to_mark->ref = false; 1133 } else { 1134 DVLOG(1) << "Invalid short ref pic num to unmark"; 1135 return false; 1136 } 1137 break; 1138 1139 case 2: 1140 // Mark a long term reference picture as unused so it can be removed 1141 // if outputted. 1142 to_mark = dpb_.GetLongRefPicByLongTermPicNum( 1143 ref_pic_marking->long_term_pic_num); 1144 if (to_mark) { 1145 to_mark->ref = false; 1146 } else { 1147 DVLOG(1) << "Invalid long term ref pic num to unmark"; 1148 return false; 1149 } 1150 break; 1151 1152 case 3: 1153 // Mark a short term reference picture as long term reference. 1154 pic_num_x = curr_pic_->pic_num - 1155 (ref_pic_marking->difference_of_pic_nums_minus1 + 1); 1156 to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x); 1157 if (to_mark) { 1158 DCHECK(to_mark->ref && !to_mark->long_term); 1159 to_mark->long_term = true; 1160 to_mark->long_term_frame_idx = ref_pic_marking->long_term_frame_idx; 1161 } else { 1162 DVLOG(1) << "Invalid short term ref pic num to mark as long ref"; 1163 return false; 1164 } 1165 break; 1166 1167 case 4: { 1168 // Unmark all reference pictures with long_term_frame_idx over new max. 1169 max_long_term_frame_idx_ 1170 = ref_pic_marking->max_long_term_frame_idx_plus1 - 1; 1171 H264Picture::PtrVector long_terms; 1172 dpb_.GetLongTermRefPicsAppending(long_terms); 1173 for (size_t i = 0; i < long_terms.size(); ++i) { 1174 H264Picture* pic = long_terms[i]; 1175 DCHECK(pic->ref && pic->long_term); 1176 // Ok to cast, max_long_term_frame_idx is much smaller than 16bit. 1177 if (pic->long_term_frame_idx > 1178 static_cast<int>(max_long_term_frame_idx_)) 1179 pic->ref = false; 1180 } 1181 break; 1182 } 1183 1184 case 5: 1185 // Unmark all reference pictures. 1186 dpb_.MarkAllUnusedForRef(); 1187 max_long_term_frame_idx_ = -1; 1188 curr_pic_->mem_mgmt_5 = true; 1189 break; 1190 1191 case 6: { 1192 // Replace long term reference pictures with current picture. 1193 // First unmark if any existing with this long_term_frame_idx... 1194 H264Picture::PtrVector long_terms; 1195 dpb_.GetLongTermRefPicsAppending(long_terms); 1196 for (size_t i = 0; i < long_terms.size(); ++i) { 1197 H264Picture* pic = long_terms[i]; 1198 DCHECK(pic->ref && pic->long_term); 1199 // Ok to cast, long_term_frame_idx is much smaller than 16bit. 1200 if (pic->long_term_frame_idx == 1201 static_cast<int>(ref_pic_marking->long_term_frame_idx)) 1202 pic->ref = false; 1203 } 1204 1205 // and mark the current one instead. 1206 curr_pic_->ref = true; 1207 curr_pic_->long_term = true; 1208 curr_pic_->long_term_frame_idx = ref_pic_marking->long_term_frame_idx; 1209 break; 1210 } 1211 1212 default: 1213 // Would indicate a bug in parser. 1214 NOTREACHED(); 1215 } 1216 } 1217 1218 return true; 1219 } 1220 1221 // This method ensures that DPB does not overflow, either by removing 1222 // reference pictures as specified in the stream, or using a sliding window 1223 // procedure to remove the oldest one. 1224 // It also performs marking and unmarking pictures as reference. 1225 // See spac 8.2.5.1. 1226 void VaapiH264Decoder::ReferencePictureMarking() { 1227 if (curr_pic_->idr) { 1228 // If current picture is an IDR, all reference pictures are unmarked. 1229 dpb_.MarkAllUnusedForRef(); 1230 1231 if (curr_pic_->long_term_reference_flag) { 1232 curr_pic_->long_term = true; 1233 curr_pic_->long_term_frame_idx = 0; 1234 max_long_term_frame_idx_ = 0; 1235 } else { 1236 curr_pic_->long_term = false; 1237 max_long_term_frame_idx_ = -1; 1238 } 1239 } else { 1240 if (!curr_pic_->adaptive_ref_pic_marking_mode_flag) { 1241 // If non-IDR, and the stream does not indicate what we should do to 1242 // ensure DPB doesn't overflow, discard oldest picture. 1243 // See spec 8.2.5.3. 1244 if (curr_pic_->field == H264Picture::FIELD_NONE) { 1245 DCHECK_LE(dpb_.CountRefPics(), 1246 std::max<int>(parser_.GetSPS(curr_sps_id_)->max_num_ref_frames, 1247 1)); 1248 if (dpb_.CountRefPics() == 1249 std::max<int>(parser_.GetSPS(curr_sps_id_)->max_num_ref_frames, 1250 1)) { 1251 // Max number of reference pics reached, 1252 // need to remove one of the short term ones. 1253 // Find smallest frame_num_wrap short reference picture and mark 1254 // it as unused. 1255 H264Picture* to_unmark = dpb_.GetLowestFrameNumWrapShortRefPic(); 1256 if (to_unmark == NULL) { 1257 DVLOG(1) << "Couldn't find a short ref picture to unmark"; 1258 return; 1259 } 1260 to_unmark->ref = false; 1261 } 1262 } else { 1263 // Shouldn't get here. 1264 DVLOG(1) << "Interlaced video not supported."; 1265 report_error_to_uma_cb_.Run(INTERLACED_STREAM); 1266 } 1267 } else { 1268 // Stream has instructions how to discard pictures from DPB and how 1269 // to mark/unmark existing reference pictures. Do it. 1270 // Spec 8.2.5.4. 1271 if (curr_pic_->field == H264Picture::FIELD_NONE) { 1272 HandleMemoryManagementOps(); 1273 } else { 1274 // Shouldn't get here. 1275 DVLOG(1) << "Interlaced video not supported."; 1276 report_error_to_uma_cb_.Run(INTERLACED_STREAM); 1277 } 1278 } 1279 } 1280 } 1281 1282 bool VaapiH264Decoder::FinishPicture() { 1283 DCHECK(curr_pic_.get()); 1284 1285 // Finish processing previous picture. 1286 // Start by storing previous reference picture data for later use, 1287 // if picture being finished is a reference picture. 1288 if (curr_pic_->ref) { 1289 ReferencePictureMarking(); 1290 prev_ref_has_memmgmnt5_ = curr_pic_->mem_mgmt_5; 1291 prev_ref_top_field_order_cnt_ = curr_pic_->top_field_order_cnt; 1292 prev_ref_pic_order_cnt_msb_ = curr_pic_->pic_order_cnt_msb; 1293 prev_ref_pic_order_cnt_lsb_ = curr_pic_->pic_order_cnt_lsb; 1294 prev_ref_field_ = curr_pic_->field; 1295 } 1296 prev_has_memmgmnt5_ = curr_pic_->mem_mgmt_5; 1297 prev_frame_num_offset_ = curr_pic_->frame_num_offset; 1298 1299 // Remove unused (for reference or later output) pictures from DPB, marking 1300 // them as such. 1301 for (H264DPB::Pictures::iterator it = dpb_.begin(); it != dpb_.end(); ++it) { 1302 if ((*it)->outputted && !(*it)->ref) 1303 UnassignSurfaceFromPoC((*it)->pic_order_cnt); 1304 } 1305 dpb_.DeleteUnused(); 1306 1307 DVLOG(4) << "Finishing picture, entries in DPB: " << dpb_.size(); 1308 1309 // Whatever happens below, curr_pic_ will stop managing the pointer to the 1310 // picture after this function returns. The ownership will either be 1311 // transferred to DPB, if the image is still needed (for output and/or 1312 // reference), or the memory will be released if we manage to output it here 1313 // without having to store it for future reference. 1314 scoped_ptr<H264Picture> pic(curr_pic_.release()); 1315 1316 // Get all pictures that haven't been outputted yet. 1317 H264Picture::PtrVector not_outputted; 1318 // TODO(posciak): pass as pointer, not reference (violates coding style). 1319 dpb_.GetNotOutputtedPicsAppending(not_outputted); 1320 // Include the one we've just decoded. 1321 not_outputted.push_back(pic.get()); 1322 // Sort in output order. 1323 std::sort(not_outputted.begin(), not_outputted.end(), POCAscCompare()); 1324 1325 // Try to output as many pictures as we can. A picture can be output 1326 // if its POC is next after the previously outputted one (which means 1327 // last_output_poc_ + 2, because POCs are incremented by 2 to accommodate 1328 // fields when decoding interleaved streams). POC can also be equal to 1329 // last outputted picture's POC when it wraps around back to 0. 1330 // If the outputted picture is not a reference picture, it doesn't have 1331 // to remain in the DPB and can be removed. 1332 H264Picture::PtrVector::iterator output_candidate = not_outputted.begin(); 1333 for (; output_candidate != not_outputted.end() && 1334 (*output_candidate)->pic_order_cnt <= last_output_poc_ + 2; 1335 ++output_candidate) { 1336 DCHECK_GE((*output_candidate)->pic_order_cnt, last_output_poc_); 1337 if (!OutputPic(*output_candidate)) 1338 return false; 1339 1340 if (!(*output_candidate)->ref) { 1341 // Current picture hasn't been inserted into DPB yet, so don't remove it 1342 // if we managed to output it immediately. 1343 if (*output_candidate != pic) 1344 dpb_.DeleteByPOC((*output_candidate)->pic_order_cnt); 1345 // Mark as unused. 1346 UnassignSurfaceFromPoC((*output_candidate)->pic_order_cnt); 1347 } 1348 } 1349 1350 // If we haven't managed to output the picture that we just decoded, or if 1351 // it's a reference picture, we have to store it in DPB. 1352 if (!pic->outputted || pic->ref) { 1353 if (dpb_.IsFull()) { 1354 // If we haven't managed to output anything to free up space in DPB 1355 // to store this picture, it's an error in the stream. 1356 DVLOG(1) << "Could not free up space in DPB!"; 1357 return false; 1358 } 1359 1360 dpb_.StorePic(pic.release()); 1361 } 1362 1363 return true; 1364 } 1365 1366 static int LevelToMaxDpbMbs(int level) { 1367 // See table A-1 in spec. 1368 switch (level) { 1369 case 10: return 396; 1370 case 11: return 900; 1371 case 12: // fallthrough 1372 case 13: // fallthrough 1373 case 20: return 2376; 1374 case 21: return 4752; 1375 case 22: // fallthrough 1376 case 30: return 8100; 1377 case 31: return 18000; 1378 case 32: return 20480; 1379 case 40: // fallthrough 1380 case 41: return 32768; 1381 case 42: return 34816; 1382 case 50: return 110400; 1383 case 51: // fallthrough 1384 case 52: return 184320; 1385 default: 1386 DVLOG(1) << "Invalid codec level (" << level << ")"; 1387 return 0; 1388 } 1389 } 1390 1391 bool VaapiH264Decoder::ProcessSPS(int sps_id, bool* need_new_buffers) { 1392 const H264SPS* sps = parser_.GetSPS(sps_id); 1393 DCHECK(sps); 1394 DVLOG(4) << "Processing SPS"; 1395 1396 *need_new_buffers = false; 1397 1398 if (sps->frame_mbs_only_flag == 0) { 1399 DVLOG(1) << "frame_mbs_only_flag != 1 not supported"; 1400 report_error_to_uma_cb_.Run(FRAME_MBS_ONLY_FLAG_NOT_ONE); 1401 return false; 1402 } 1403 1404 if (sps->gaps_in_frame_num_value_allowed_flag) { 1405 DVLOG(1) << "Gaps in frame numbers not supported"; 1406 report_error_to_uma_cb_.Run(GAPS_IN_FRAME_NUM); 1407 return false; 1408 } 1409 1410 curr_sps_id_ = sps->seq_parameter_set_id; 1411 1412 // Calculate picture height/width in macroblocks and pixels 1413 // (spec 7.4.2.1.1, 7.4.3). 1414 int width_mb = sps->pic_width_in_mbs_minus1 + 1; 1415 int height_mb = (2 - sps->frame_mbs_only_flag) * 1416 (sps->pic_height_in_map_units_minus1 + 1); 1417 1418 gfx::Size new_pic_size(16 * width_mb, 16 * height_mb); 1419 if (new_pic_size.IsEmpty()) { 1420 DVLOG(1) << "Invalid picture size: " << new_pic_size.ToString(); 1421 return false; 1422 } 1423 1424 if (!pic_size_.IsEmpty() && new_pic_size == pic_size_) { 1425 // Already have surfaces and this SPS keeps the same resolution, 1426 // no need to request a new set. 1427 return true; 1428 } 1429 1430 pic_size_ = new_pic_size; 1431 DVLOG(1) << "New picture size: " << pic_size_.ToString(); 1432 1433 max_pic_order_cnt_lsb_ = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4); 1434 max_frame_num_ = 1 << (sps->log2_max_frame_num_minus4 + 4); 1435 1436 int level = sps->level_idc; 1437 int max_dpb_mbs = LevelToMaxDpbMbs(level); 1438 if (max_dpb_mbs == 0) 1439 return false; 1440 1441 size_t max_dpb_size = std::min(max_dpb_mbs / (width_mb * height_mb), 1442 static_cast<int>(H264DPB::kDPBMaxSize)); 1443 DVLOG(1) << "Codec level: " << level << ", DPB size: " << max_dpb_size; 1444 1445 dpb_.set_max_num_pics(max_dpb_size); 1446 1447 *need_new_buffers = true; 1448 return true; 1449 } 1450 1451 bool VaapiH264Decoder::ProcessPPS(int pps_id) { 1452 const H264PPS* pps = parser_.GetPPS(pps_id); 1453 DCHECK(pps); 1454 1455 curr_pps_id_ = pps->pic_parameter_set_id; 1456 1457 return true; 1458 } 1459 1460 bool VaapiH264Decoder::FinishPrevFrameIfPresent() { 1461 // If we already have a frame waiting to be decoded, decode it and finish. 1462 if (curr_pic_ != NULL) { 1463 if (!DecodePicture()) 1464 return false; 1465 return FinishPicture(); 1466 } 1467 1468 return true; 1469 } 1470 1471 bool VaapiH264Decoder::ProcessSlice(H264SliceHeader* slice_hdr) { 1472 prev_frame_num_ = frame_num_; 1473 frame_num_ = slice_hdr->frame_num; 1474 1475 if (prev_frame_num_ > 0 && prev_frame_num_ < frame_num_ - 1) { 1476 DVLOG(1) << "Gap in frame_num!"; 1477 report_error_to_uma_cb_.Run(GAPS_IN_FRAME_NUM); 1478 return false; 1479 } 1480 1481 if (slice_hdr->field_pic_flag == 0) 1482 max_pic_num_ = max_frame_num_; 1483 else 1484 max_pic_num_ = 2 * max_frame_num_; 1485 1486 // TODO posciak: switch to new picture detection per 7.4.1.2.4. 1487 if (curr_pic_ != NULL && slice_hdr->first_mb_in_slice != 0) { 1488 // This is just some more slice data of the current picture, so 1489 // just queue it and return. 1490 QueueSlice(slice_hdr); 1491 return true; 1492 } else { 1493 // A new frame, so first finish the previous one before processing it... 1494 if (!FinishPrevFrameIfPresent()) 1495 return false; 1496 1497 // and then start a new one. 1498 return StartNewFrame(slice_hdr); 1499 } 1500 } 1501 1502 #define SET_ERROR_AND_RETURN() \ 1503 do { \ 1504 DVLOG(1) << "Error during decode"; \ 1505 state_ = kError; \ 1506 return VaapiH264Decoder::kDecodeError; \ 1507 } while (0) 1508 1509 void VaapiH264Decoder::SetStream(uint8* ptr, size_t size, int32 input_id) { 1510 DCHECK(ptr); 1511 DCHECK(size); 1512 1513 // Got new input stream data from the client. 1514 DVLOG(4) << "New input stream id: " << input_id << " at: " << (void*) ptr 1515 << " size: " << size; 1516 parser_.SetStream(ptr, size); 1517 curr_input_id_ = input_id; 1518 } 1519 1520 VaapiH264Decoder::DecResult VaapiH264Decoder::Decode() { 1521 H264Parser::Result par_res; 1522 H264NALU nalu; 1523 DCHECK_NE(state_, kError); 1524 1525 while (1) { 1526 // If we've already decoded some of the stream (after reset, i.e. we are 1527 // not in kNeedStreamMetadata state), we may be able to go back into 1528 // decoding state not only starting at/resuming from an SPS, but also from 1529 // other resume points, such as IDRs. In the latter case we need an output 1530 // surface, because we will end up decoding that IDR in the process. 1531 // Otherwise we just look for an SPS and don't produce any output frames. 1532 if (state_ != kNeedStreamMetadata && available_va_surfaces_.empty()) { 1533 DVLOG(4) << "No output surfaces available"; 1534 return kRanOutOfSurfaces; 1535 } 1536 1537 par_res = parser_.AdvanceToNextNALU(&nalu); 1538 if (par_res == H264Parser::kEOStream) 1539 return kRanOutOfStreamData; 1540 else if (par_res != H264Parser::kOk) 1541 SET_ERROR_AND_RETURN(); 1542 1543 DVLOG(4) << "NALU found: " << static_cast<int>(nalu.nal_unit_type); 1544 1545 switch (nalu.nal_unit_type) { 1546 case H264NALU::kNonIDRSlice: 1547 // We can't resume from a non-IDR slice. 1548 if (state_ != kDecoding) 1549 break; 1550 // else fallthrough 1551 case H264NALU::kIDRSlice: { 1552 // TODO(posciak): the IDR may require an SPS that we don't have 1553 // available. For now we'd fail if that happens, but ideally we'd like 1554 // to keep going until the next SPS in the stream. 1555 if (state_ == kNeedStreamMetadata) { 1556 // We need an SPS, skip this IDR and keep looking. 1557 break; 1558 } 1559 1560 // If after reset, we should be able to recover from an IDR. 1561 H264SliceHeader slice_hdr; 1562 1563 par_res = parser_.ParseSliceHeader(nalu, &slice_hdr); 1564 if (par_res != H264Parser::kOk) 1565 SET_ERROR_AND_RETURN(); 1566 1567 if (!ProcessSlice(&slice_hdr)) 1568 SET_ERROR_AND_RETURN(); 1569 1570 state_ = kDecoding; 1571 break; 1572 } 1573 1574 case H264NALU::kSPS: { 1575 int sps_id; 1576 1577 if (!FinishPrevFrameIfPresent()) 1578 SET_ERROR_AND_RETURN(); 1579 1580 par_res = parser_.ParseSPS(&sps_id); 1581 if (par_res != H264Parser::kOk) 1582 SET_ERROR_AND_RETURN(); 1583 1584 bool need_new_buffers = false; 1585 if (!ProcessSPS(sps_id, &need_new_buffers)) 1586 SET_ERROR_AND_RETURN(); 1587 1588 state_ = kDecoding; 1589 1590 if (need_new_buffers) { 1591 if (!Flush()) 1592 return kDecodeError; 1593 1594 available_va_surfaces_.clear(); 1595 return kAllocateNewSurfaces; 1596 } 1597 break; 1598 } 1599 1600 case H264NALU::kPPS: { 1601 if (state_ != kDecoding) 1602 break; 1603 1604 int pps_id; 1605 1606 if (!FinishPrevFrameIfPresent()) 1607 SET_ERROR_AND_RETURN(); 1608 1609 par_res = parser_.ParsePPS(&pps_id); 1610 if (par_res != H264Parser::kOk) 1611 SET_ERROR_AND_RETURN(); 1612 1613 if (!ProcessPPS(pps_id)) 1614 SET_ERROR_AND_RETURN(); 1615 break; 1616 } 1617 1618 default: 1619 DVLOG(4) << "Skipping NALU type: " << nalu.nal_unit_type;; 1620 break; 1621 } 1622 } 1623 } 1624 1625 size_t VaapiH264Decoder::GetRequiredNumOfPictures() { 1626 return dpb_.max_num_pics() + kPicsInPipeline; 1627 } 1628 1629 } // namespace content 1630
