1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "webrtc/modules/video_coding/session_info.h" 12 13 #include "webrtc/base/logging.h" 14 #include "webrtc/modules/video_coding/packet.h" 15 16 namespace webrtc { 17 18 namespace { 19 20 uint16_t BufferToUWord16(const uint8_t* dataBuffer) { 21 return (dataBuffer[0] << 8) | dataBuffer[1]; 22 } 23 24 } // namespace 25 26 VCMSessionInfo::VCMSessionInfo() 27 : session_nack_(false), 28 complete_(false), 29 decodable_(false), 30 frame_type_(kVideoFrameDelta), 31 packets_(), 32 empty_seq_num_low_(-1), 33 empty_seq_num_high_(-1), 34 first_packet_seq_num_(-1), 35 last_packet_seq_num_(-1) {} 36 37 void VCMSessionInfo::UpdateDataPointers(const uint8_t* old_base_ptr, 38 const uint8_t* new_base_ptr) { 39 for (PacketIterator it = packets_.begin(); it != packets_.end(); ++it) 40 if ((*it).dataPtr != NULL) { 41 assert(old_base_ptr != NULL && new_base_ptr != NULL); 42 (*it).dataPtr = new_base_ptr + ((*it).dataPtr - old_base_ptr); 43 } 44 } 45 46 int VCMSessionInfo::LowSequenceNumber() const { 47 if (packets_.empty()) 48 return empty_seq_num_low_; 49 return packets_.front().seqNum; 50 } 51 52 int VCMSessionInfo::HighSequenceNumber() const { 53 if (packets_.empty()) 54 return empty_seq_num_high_; 55 if (empty_seq_num_high_ == -1) 56 return packets_.back().seqNum; 57 return LatestSequenceNumber(packets_.back().seqNum, empty_seq_num_high_); 58 } 59 60 int VCMSessionInfo::PictureId() const { 61 if (packets_.empty()) 62 return kNoPictureId; 63 if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp8) { 64 return packets_.front().codecSpecificHeader.codecHeader.VP8.pictureId; 65 } else if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp9) { 66 return packets_.front().codecSpecificHeader.codecHeader.VP9.picture_id; 67 } else { 68 return kNoPictureId; 69 } 70 } 71 72 int VCMSessionInfo::TemporalId() const { 73 if (packets_.empty()) 74 return kNoTemporalIdx; 75 if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp8) { 76 return packets_.front().codecSpecificHeader.codecHeader.VP8.temporalIdx; 77 } else if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp9) { 78 return packets_.front().codecSpecificHeader.codecHeader.VP9.temporal_idx; 79 } else { 80 return kNoTemporalIdx; 81 } 82 } 83 84 bool VCMSessionInfo::LayerSync() const { 85 if (packets_.empty()) 86 return false; 87 if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp8) { 88 return packets_.front().codecSpecificHeader.codecHeader.VP8.layerSync; 89 } else if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp9) { 90 return packets_.front() 91 .codecSpecificHeader.codecHeader.VP9.temporal_up_switch; 92 } else { 93 return false; 94 } 95 } 96 97 int VCMSessionInfo::Tl0PicId() const { 98 if (packets_.empty()) 99 return kNoTl0PicIdx; 100 if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp8) { 101 return packets_.front().codecSpecificHeader.codecHeader.VP8.tl0PicIdx; 102 } else if (packets_.front().codecSpecificHeader.codec == kRtpVideoVp9) { 103 return packets_.front().codecSpecificHeader.codecHeader.VP9.tl0_pic_idx; 104 } else { 105 return kNoTl0PicIdx; 106 } 107 } 108 109 bool VCMSessionInfo::NonReference() const { 110 if (packets_.empty() || 111 packets_.front().codecSpecificHeader.codec != kRtpVideoVp8) 112 return false; 113 return packets_.front().codecSpecificHeader.codecHeader.VP8.nonReference; 114 } 115 116 void VCMSessionInfo::SetGofInfo(const GofInfoVP9& gof_info, size_t idx) { 117 if (packets_.empty() || 118 packets_.front().codecSpecificHeader.codec != kRtpVideoVp9 || 119 packets_.front().codecSpecificHeader.codecHeader.VP9.flexible_mode) { 120 return; 121 } 122 packets_.front().codecSpecificHeader.codecHeader.VP9.temporal_idx = 123 gof_info.temporal_idx[idx]; 124 packets_.front().codecSpecificHeader.codecHeader.VP9.temporal_up_switch = 125 gof_info.temporal_up_switch[idx]; 126 packets_.front().codecSpecificHeader.codecHeader.VP9.num_ref_pics = 127 gof_info.num_ref_pics[idx]; 128 for (uint8_t i = 0; i < gof_info.num_ref_pics[idx]; ++i) { 129 packets_.front().codecSpecificHeader.codecHeader.VP9.pid_diff[i] = 130 gof_info.pid_diff[idx][i]; 131 } 132 } 133 134 void VCMSessionInfo::Reset() { 135 session_nack_ = false; 136 complete_ = false; 137 decodable_ = false; 138 frame_type_ = kVideoFrameDelta; 139 packets_.clear(); 140 empty_seq_num_low_ = -1; 141 empty_seq_num_high_ = -1; 142 first_packet_seq_num_ = -1; 143 last_packet_seq_num_ = -1; 144 } 145 146 size_t VCMSessionInfo::SessionLength() const { 147 size_t length = 0; 148 for (PacketIteratorConst it = packets_.begin(); it != packets_.end(); ++it) 149 length += (*it).sizeBytes; 150 return length; 151 } 152 153 int VCMSessionInfo::NumPackets() const { 154 return packets_.size(); 155 } 156 157 size_t VCMSessionInfo::InsertBuffer(uint8_t* frame_buffer, 158 PacketIterator packet_it) { 159 VCMPacket& packet = *packet_it; 160 PacketIterator it; 161 162 // Calculate the offset into the frame buffer for this packet. 163 size_t offset = 0; 164 for (it = packets_.begin(); it != packet_it; ++it) 165 offset += (*it).sizeBytes; 166 167 // Set the data pointer to pointing to the start of this packet in the 168 // frame buffer. 169 const uint8_t* packet_buffer = packet.dataPtr; 170 packet.dataPtr = frame_buffer + offset; 171 172 // We handle H.264 STAP-A packets in a special way as we need to remove the 173 // two length bytes between each NAL unit, and potentially add start codes. 174 // TODO(pbos): Remove H264 parsing from this step and use a fragmentation 175 // header supplied by the H264 depacketizer. 176 const size_t kH264NALHeaderLengthInBytes = 1; 177 const size_t kLengthFieldLength = 2; 178 if (packet.codecSpecificHeader.codec == kRtpVideoH264 && 179 packet.codecSpecificHeader.codecHeader.H264.packetization_type == 180 kH264StapA) { 181 size_t required_length = 0; 182 const uint8_t* nalu_ptr = packet_buffer + kH264NALHeaderLengthInBytes; 183 while (nalu_ptr < packet_buffer + packet.sizeBytes) { 184 size_t length = BufferToUWord16(nalu_ptr); 185 required_length += 186 length + (packet.insertStartCode ? kH264StartCodeLengthBytes : 0); 187 nalu_ptr += kLengthFieldLength + length; 188 } 189 ShiftSubsequentPackets(packet_it, required_length); 190 nalu_ptr = packet_buffer + kH264NALHeaderLengthInBytes; 191 uint8_t* frame_buffer_ptr = frame_buffer + offset; 192 while (nalu_ptr < packet_buffer + packet.sizeBytes) { 193 size_t length = BufferToUWord16(nalu_ptr); 194 nalu_ptr += kLengthFieldLength; 195 frame_buffer_ptr += Insert(nalu_ptr, length, packet.insertStartCode, 196 const_cast<uint8_t*>(frame_buffer_ptr)); 197 nalu_ptr += length; 198 } 199 packet.sizeBytes = required_length; 200 return packet.sizeBytes; 201 } 202 ShiftSubsequentPackets( 203 packet_it, packet.sizeBytes + 204 (packet.insertStartCode ? kH264StartCodeLengthBytes : 0)); 205 206 packet.sizeBytes = 207 Insert(packet_buffer, packet.sizeBytes, packet.insertStartCode, 208 const_cast<uint8_t*>(packet.dataPtr)); 209 return packet.sizeBytes; 210 } 211 212 size_t VCMSessionInfo::Insert(const uint8_t* buffer, 213 size_t length, 214 bool insert_start_code, 215 uint8_t* frame_buffer) { 216 if (insert_start_code) { 217 const unsigned char startCode[] = {0, 0, 0, 1}; 218 memcpy(frame_buffer, startCode, kH264StartCodeLengthBytes); 219 } 220 memcpy(frame_buffer + (insert_start_code ? kH264StartCodeLengthBytes : 0), 221 buffer, length); 222 length += (insert_start_code ? kH264StartCodeLengthBytes : 0); 223 224 return length; 225 } 226 227 void VCMSessionInfo::ShiftSubsequentPackets(PacketIterator it, 228 int steps_to_shift) { 229 ++it; 230 if (it == packets_.end()) 231 return; 232 uint8_t* first_packet_ptr = const_cast<uint8_t*>((*it).dataPtr); 233 int shift_length = 0; 234 // Calculate the total move length and move the data pointers in advance. 235 for (; it != packets_.end(); ++it) { 236 shift_length += (*it).sizeBytes; 237 if ((*it).dataPtr != NULL) 238 (*it).dataPtr += steps_to_shift; 239 } 240 memmove(first_packet_ptr + steps_to_shift, first_packet_ptr, shift_length); 241 } 242 243 void VCMSessionInfo::UpdateCompleteSession() { 244 if (HaveFirstPacket() && HaveLastPacket()) { 245 // Do we have all the packets in this session? 246 bool complete_session = true; 247 PacketIterator it = packets_.begin(); 248 PacketIterator prev_it = it; 249 ++it; 250 for (; it != packets_.end(); ++it) { 251 if (!InSequence(it, prev_it)) { 252 complete_session = false; 253 break; 254 } 255 prev_it = it; 256 } 257 complete_ = complete_session; 258 } 259 } 260 261 void VCMSessionInfo::UpdateDecodableSession(const FrameData& frame_data) { 262 // Irrelevant if session is already complete or decodable 263 if (complete_ || decodable_) 264 return; 265 // TODO(agalusza): Account for bursty loss. 266 // TODO(agalusza): Refine these values to better approximate optimal ones. 267 // Do not decode frames if the RTT is lower than this. 268 const int64_t kRttThreshold = 100; 269 // Do not decode frames if the number of packets is between these two 270 // thresholds. 271 const float kLowPacketPercentageThreshold = 0.2f; 272 const float kHighPacketPercentageThreshold = 0.8f; 273 if (frame_data.rtt_ms < kRttThreshold || frame_type_ == kVideoFrameKey || 274 !HaveFirstPacket() || 275 (NumPackets() <= kHighPacketPercentageThreshold * 276 frame_data.rolling_average_packets_per_frame && 277 NumPackets() > kLowPacketPercentageThreshold * 278 frame_data.rolling_average_packets_per_frame)) 279 return; 280 281 decodable_ = true; 282 } 283 284 bool VCMSessionInfo::complete() const { 285 return complete_; 286 } 287 288 bool VCMSessionInfo::decodable() const { 289 return decodable_; 290 } 291 292 // Find the end of the NAL unit which the packet pointed to by |packet_it| 293 // belongs to. Returns an iterator to the last packet of the frame if the end 294 // of the NAL unit wasn't found. 295 VCMSessionInfo::PacketIterator VCMSessionInfo::FindNaluEnd( 296 PacketIterator packet_it) const { 297 if ((*packet_it).completeNALU == kNaluEnd || 298 (*packet_it).completeNALU == kNaluComplete) { 299 return packet_it; 300 } 301 // Find the end of the NAL unit. 302 for (; packet_it != packets_.end(); ++packet_it) { 303 if (((*packet_it).completeNALU == kNaluComplete && 304 (*packet_it).sizeBytes > 0) || 305 // Found next NALU. 306 (*packet_it).completeNALU == kNaluStart) 307 return --packet_it; 308 if ((*packet_it).completeNALU == kNaluEnd) 309 return packet_it; 310 } 311 // The end wasn't found. 312 return --packet_it; 313 } 314 315 size_t VCMSessionInfo::DeletePacketData(PacketIterator start, 316 PacketIterator end) { 317 size_t bytes_to_delete = 0; // The number of bytes to delete. 318 PacketIterator packet_after_end = end; 319 ++packet_after_end; 320 321 // Get the number of bytes to delete. 322 // Clear the size of these packets. 323 for (PacketIterator it = start; it != packet_after_end; ++it) { 324 bytes_to_delete += (*it).sizeBytes; 325 (*it).sizeBytes = 0; 326 (*it).dataPtr = NULL; 327 } 328 if (bytes_to_delete > 0) 329 ShiftSubsequentPackets(end, -static_cast<int>(bytes_to_delete)); 330 return bytes_to_delete; 331 } 332 333 size_t VCMSessionInfo::BuildVP8FragmentationHeader( 334 uint8_t* frame_buffer, 335 size_t frame_buffer_length, 336 RTPFragmentationHeader* fragmentation) { 337 size_t new_length = 0; 338 // Allocate space for max number of partitions 339 fragmentation->VerifyAndAllocateFragmentationHeader(kMaxVP8Partitions); 340 fragmentation->fragmentationVectorSize = 0; 341 memset(fragmentation->fragmentationLength, 0, 342 kMaxVP8Partitions * sizeof(size_t)); 343 if (packets_.empty()) 344 return new_length; 345 PacketIterator it = FindNextPartitionBeginning(packets_.begin()); 346 while (it != packets_.end()) { 347 const int partition_id = 348 (*it).codecSpecificHeader.codecHeader.VP8.partitionId; 349 PacketIterator partition_end = FindPartitionEnd(it); 350 fragmentation->fragmentationOffset[partition_id] = 351 (*it).dataPtr - frame_buffer; 352 assert(fragmentation->fragmentationOffset[partition_id] < 353 frame_buffer_length); 354 fragmentation->fragmentationLength[partition_id] = 355 (*partition_end).dataPtr + (*partition_end).sizeBytes - (*it).dataPtr; 356 assert(fragmentation->fragmentationLength[partition_id] <= 357 frame_buffer_length); 358 new_length += fragmentation->fragmentationLength[partition_id]; 359 ++partition_end; 360 it = FindNextPartitionBeginning(partition_end); 361 if (partition_id + 1 > fragmentation->fragmentationVectorSize) 362 fragmentation->fragmentationVectorSize = partition_id + 1; 363 } 364 // Set all empty fragments to start where the previous fragment ends, 365 // and have zero length. 366 if (fragmentation->fragmentationLength[0] == 0) 367 fragmentation->fragmentationOffset[0] = 0; 368 for (int i = 1; i < fragmentation->fragmentationVectorSize; ++i) { 369 if (fragmentation->fragmentationLength[i] == 0) 370 fragmentation->fragmentationOffset[i] = 371 fragmentation->fragmentationOffset[i - 1] + 372 fragmentation->fragmentationLength[i - 1]; 373 assert(i == 0 || 374 fragmentation->fragmentationOffset[i] >= 375 fragmentation->fragmentationOffset[i - 1]); 376 } 377 assert(new_length <= frame_buffer_length); 378 return new_length; 379 } 380 381 VCMSessionInfo::PacketIterator VCMSessionInfo::FindNextPartitionBeginning( 382 PacketIterator it) const { 383 while (it != packets_.end()) { 384 if ((*it).codecSpecificHeader.codecHeader.VP8.beginningOfPartition) { 385 return it; 386 } 387 ++it; 388 } 389 return it; 390 } 391 392 VCMSessionInfo::PacketIterator VCMSessionInfo::FindPartitionEnd( 393 PacketIterator it) const { 394 assert((*it).codec == kVideoCodecVP8); 395 PacketIterator prev_it = it; 396 const int partition_id = 397 (*it).codecSpecificHeader.codecHeader.VP8.partitionId; 398 while (it != packets_.end()) { 399 bool beginning = 400 (*it).codecSpecificHeader.codecHeader.VP8.beginningOfPartition; 401 int current_partition_id = 402 (*it).codecSpecificHeader.codecHeader.VP8.partitionId; 403 bool packet_loss_found = (!beginning && !InSequence(it, prev_it)); 404 if (packet_loss_found || 405 (beginning && current_partition_id != partition_id)) { 406 // Missing packet, the previous packet was the last in sequence. 407 return prev_it; 408 } 409 prev_it = it; 410 ++it; 411 } 412 return prev_it; 413 } 414 415 bool VCMSessionInfo::InSequence(const PacketIterator& packet_it, 416 const PacketIterator& prev_packet_it) { 417 // If the two iterators are pointing to the same packet they are considered 418 // to be in sequence. 419 return (packet_it == prev_packet_it || 420 (static_cast<uint16_t>((*prev_packet_it).seqNum + 1) == 421 (*packet_it).seqNum)); 422 } 423 424 size_t VCMSessionInfo::MakeDecodable() { 425 size_t return_length = 0; 426 if (packets_.empty()) { 427 return 0; 428 } 429 PacketIterator it = packets_.begin(); 430 // Make sure we remove the first NAL unit if it's not decodable. 431 if ((*it).completeNALU == kNaluIncomplete || (*it).completeNALU == kNaluEnd) { 432 PacketIterator nalu_end = FindNaluEnd(it); 433 return_length += DeletePacketData(it, nalu_end); 434 it = nalu_end; 435 } 436 PacketIterator prev_it = it; 437 // Take care of the rest of the NAL units. 438 for (; it != packets_.end(); ++it) { 439 bool start_of_nalu = ((*it).completeNALU == kNaluStart || 440 (*it).completeNALU == kNaluComplete); 441 if (!start_of_nalu && !InSequence(it, prev_it)) { 442 // Found a sequence number gap due to packet loss. 443 PacketIterator nalu_end = FindNaluEnd(it); 444 return_length += DeletePacketData(it, nalu_end); 445 it = nalu_end; 446 } 447 prev_it = it; 448 } 449 return return_length; 450 } 451 452 void VCMSessionInfo::SetNotDecodableIfIncomplete() { 453 // We don't need to check for completeness first because the two are 454 // orthogonal. If complete_ is true, decodable_ is irrelevant. 455 decodable_ = false; 456 } 457 458 bool VCMSessionInfo::HaveFirstPacket() const { 459 return !packets_.empty() && (first_packet_seq_num_ != -1); 460 } 461 462 bool VCMSessionInfo::HaveLastPacket() const { 463 return !packets_.empty() && (last_packet_seq_num_ != -1); 464 } 465 466 bool VCMSessionInfo::session_nack() const { 467 return session_nack_; 468 } 469 470 int VCMSessionInfo::InsertPacket(const VCMPacket& packet, 471 uint8_t* frame_buffer, 472 VCMDecodeErrorMode decode_error_mode, 473 const FrameData& frame_data) { 474 if (packet.frameType == kEmptyFrame) { 475 // Update sequence number of an empty packet. 476 // Only media packets are inserted into the packet list. 477 InformOfEmptyPacket(packet.seqNum); 478 return 0; 479 } 480 481 if (packets_.size() == kMaxPacketsInSession) { 482 LOG(LS_ERROR) << "Max number of packets per frame has been reached."; 483 return -1; 484 } 485 486 // Find the position of this packet in the packet list in sequence number 487 // order and insert it. Loop over the list in reverse order. 488 ReversePacketIterator rit = packets_.rbegin(); 489 for (; rit != packets_.rend(); ++rit) 490 if (LatestSequenceNumber(packet.seqNum, (*rit).seqNum) == packet.seqNum) 491 break; 492 493 // Check for duplicate packets. 494 if (rit != packets_.rend() && (*rit).seqNum == packet.seqNum && 495 (*rit).sizeBytes > 0) 496 return -2; 497 498 if (packet.codec == kVideoCodecH264) { 499 frame_type_ = packet.frameType; 500 if (packet.isFirstPacket && 501 (first_packet_seq_num_ == -1 || 502 IsNewerSequenceNumber(first_packet_seq_num_, packet.seqNum))) { 503 first_packet_seq_num_ = packet.seqNum; 504 } 505 if (packet.markerBit && 506 (last_packet_seq_num_ == -1 || 507 IsNewerSequenceNumber(packet.seqNum, last_packet_seq_num_))) { 508 last_packet_seq_num_ = packet.seqNum; 509 } 510 } else { 511 // Only insert media packets between first and last packets (when 512 // available). 513 // Placing check here, as to properly account for duplicate packets. 514 // Check if this is first packet (only valid for some codecs) 515 // Should only be set for one packet per session. 516 if (packet.isFirstPacket && first_packet_seq_num_ == -1) { 517 // The first packet in a frame signals the frame type. 518 frame_type_ = packet.frameType; 519 // Store the sequence number for the first packet. 520 first_packet_seq_num_ = static_cast<int>(packet.seqNum); 521 } else if (first_packet_seq_num_ != -1 && 522 IsNewerSequenceNumber(first_packet_seq_num_, packet.seqNum)) { 523 LOG(LS_WARNING) << "Received packet with a sequence number which is out " 524 "of frame boundaries"; 525 return -3; 526 } else if (frame_type_ == kEmptyFrame && packet.frameType != kEmptyFrame) { 527 // Update the frame type with the type of the first media packet. 528 // TODO(mikhal): Can this trigger? 529 frame_type_ = packet.frameType; 530 } 531 532 // Track the marker bit, should only be set for one packet per session. 533 if (packet.markerBit && last_packet_seq_num_ == -1) { 534 last_packet_seq_num_ = static_cast<int>(packet.seqNum); 535 } else if (last_packet_seq_num_ != -1 && 536 IsNewerSequenceNumber(packet.seqNum, last_packet_seq_num_)) { 537 LOG(LS_WARNING) << "Received packet with a sequence number which is out " 538 "of frame boundaries"; 539 return -3; 540 } 541 } 542 543 // The insert operation invalidates the iterator |rit|. 544 PacketIterator packet_list_it = packets_.insert(rit.base(), packet); 545 546 size_t returnLength = InsertBuffer(frame_buffer, packet_list_it); 547 UpdateCompleteSession(); 548 if (decode_error_mode == kWithErrors) 549 decodable_ = true; 550 else if (decode_error_mode == kSelectiveErrors) 551 UpdateDecodableSession(frame_data); 552 return static_cast<int>(returnLength); 553 } 554 555 void VCMSessionInfo::InformOfEmptyPacket(uint16_t seq_num) { 556 // Empty packets may be FEC or filler packets. They are sequential and 557 // follow the data packets, therefore, we should only keep track of the high 558 // and low sequence numbers and may assume that the packets in between are 559 // empty packets belonging to the same frame (timestamp). 560 if (empty_seq_num_high_ == -1) 561 empty_seq_num_high_ = seq_num; 562 else 563 empty_seq_num_high_ = LatestSequenceNumber(seq_num, empty_seq_num_high_); 564 if (empty_seq_num_low_ == -1 || 565 IsNewerSequenceNumber(empty_seq_num_low_, seq_num)) 566 empty_seq_num_low_ = seq_num; 567 } 568 569 } // namespace webrtc 570
