1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "webrtc/modules/video_coding/main/source/session_info.h" 12 13 #include "webrtc/modules/video_coding/main/source/packet.h" 14 #include "webrtc/system_wrappers/interface/logging.h" 15 16 namespace webrtc { 17 18 // Used in determining whether a frame is decodable. 19 enum {kRttThreshold = 100}; // Not decodable if Rtt is lower than this. 20 21 // Do not decode frames if the number of packets is between these two 22 // thresholds. 23 static const float kLowPacketPercentageThreshold = 0.2f; 24 static const float kHighPacketPercentageThreshold = 0.8f; 25 26 VCMSessionInfo::VCMSessionInfo() 27 : session_nack_(false), 28 complete_(false), 29 decodable_(false), 30 frame_type_(kVideoFrameDelta), 31 packets_(), 32 empty_seq_num_low_(-1), 33 empty_seq_num_high_(-1), 34 first_packet_seq_num_(-1), 35 last_packet_seq_num_(-1) { 36 } 37 38 void VCMSessionInfo::UpdateDataPointers(const uint8_t* old_base_ptr, 39 const uint8_t* new_base_ptr) { 40 for (PacketIterator it = packets_.begin(); it != packets_.end(); ++it) 41 if ((*it).dataPtr != NULL) { 42 assert(old_base_ptr != NULL && new_base_ptr != NULL); 43 (*it).dataPtr = new_base_ptr + ((*it).dataPtr - old_base_ptr); 44 } 45 } 46 47 int VCMSessionInfo::LowSequenceNumber() const { 48 if (packets_.empty()) 49 return empty_seq_num_low_; 50 return packets_.front().seqNum; 51 } 52 53 int VCMSessionInfo::HighSequenceNumber() const { 54 if (packets_.empty()) 55 return empty_seq_num_high_; 56 if (empty_seq_num_high_ == -1) 57 return packets_.back().seqNum; 58 return LatestSequenceNumber(packets_.back().seqNum, empty_seq_num_high_); 59 } 60 61 int VCMSessionInfo::PictureId() const { 62 if (packets_.empty() || 63 packets_.front().codecSpecificHeader.codec != kRtpVideoVp8) 64 return kNoPictureId; 65 return packets_.front().codecSpecificHeader.codecHeader.VP8.pictureId; 66 } 67 68 int VCMSessionInfo::TemporalId() const { 69 if (packets_.empty() || 70 packets_.front().codecSpecificHeader.codec != kRtpVideoVp8) 71 return kNoTemporalIdx; 72 return packets_.front().codecSpecificHeader.codecHeader.VP8.temporalIdx; 73 } 74 75 bool VCMSessionInfo::LayerSync() const { 76 if (packets_.empty() || 77 packets_.front().codecSpecificHeader.codec != kRtpVideoVp8) 78 return false; 79 return packets_.front().codecSpecificHeader.codecHeader.VP8.layerSync; 80 } 81 82 int VCMSessionInfo::Tl0PicId() const { 83 if (packets_.empty() || 84 packets_.front().codecSpecificHeader.codec != kRtpVideoVp8) 85 return kNoTl0PicIdx; 86 return packets_.front().codecSpecificHeader.codecHeader.VP8.tl0PicIdx; 87 } 88 89 bool VCMSessionInfo::NonReference() const { 90 if (packets_.empty() || 91 packets_.front().codecSpecificHeader.codec != kRtpVideoVp8) 92 return false; 93 return packets_.front().codecSpecificHeader.codecHeader.VP8.nonReference; 94 } 95 96 void VCMSessionInfo::Reset() { 97 session_nack_ = false; 98 complete_ = false; 99 decodable_ = false; 100 frame_type_ = kVideoFrameDelta; 101 packets_.clear(); 102 empty_seq_num_low_ = -1; 103 empty_seq_num_high_ = -1; 104 first_packet_seq_num_ = -1; 105 last_packet_seq_num_ = -1; 106 } 107 108 int VCMSessionInfo::SessionLength() const { 109 int length = 0; 110 for (PacketIteratorConst it = packets_.begin(); it != packets_.end(); ++it) 111 length += (*it).sizeBytes; 112 return length; 113 } 114 115 int VCMSessionInfo::NumPackets() const { 116 return packets_.size(); 117 } 118 119 int VCMSessionInfo::InsertBuffer(uint8_t* frame_buffer, 120 PacketIterator packet_it) { 121 VCMPacket& packet = *packet_it; 122 PacketIterator it; 123 124 int packet_size = packet.sizeBytes; 125 packet_size += (packet.insertStartCode ? kH264StartCodeLengthBytes : 0); 126 127 // Calculate the offset into the frame buffer for this packet. 128 int offset = 0; 129 for (it = packets_.begin(); it != packet_it; ++it) 130 offset += (*it).sizeBytes; 131 132 // Set the data pointer to pointing to the start of this packet in the 133 // frame buffer. 134 const uint8_t* data = packet.dataPtr; 135 packet.dataPtr = frame_buffer + offset; 136 packet.sizeBytes = packet_size; 137 138 ShiftSubsequentPackets(packet_it, packet_size); 139 140 const unsigned char startCode[] = {0, 0, 0, 1}; 141 if (packet.insertStartCode) { 142 memcpy(const_cast<uint8_t*>(packet.dataPtr), startCode, 143 kH264StartCodeLengthBytes); 144 } 145 memcpy(const_cast<uint8_t*>(packet.dataPtr 146 + (packet.insertStartCode ? kH264StartCodeLengthBytes : 0)), 147 data, 148 packet.sizeBytes); 149 150 return packet_size; 151 } 152 153 void VCMSessionInfo::ShiftSubsequentPackets(PacketIterator it, 154 int steps_to_shift) { 155 ++it; 156 if (it == packets_.end()) 157 return; 158 uint8_t* first_packet_ptr = const_cast<uint8_t*>((*it).dataPtr); 159 int shift_length = 0; 160 // Calculate the total move length and move the data pointers in advance. 161 for (; it != packets_.end(); ++it) { 162 shift_length += (*it).sizeBytes; 163 if ((*it).dataPtr != NULL) 164 (*it).dataPtr += steps_to_shift; 165 } 166 memmove(first_packet_ptr + steps_to_shift, first_packet_ptr, shift_length); 167 } 168 169 void VCMSessionInfo::UpdateCompleteSession() { 170 if (HaveFirstPacket() && HaveLastPacket()) { 171 // Do we have all the packets in this session? 172 bool complete_session = true; 173 PacketIterator it = packets_.begin(); 174 PacketIterator prev_it = it; 175 ++it; 176 for (; it != packets_.end(); ++it) { 177 if (!InSequence(it, prev_it)) { 178 complete_session = false; 179 break; 180 } 181 prev_it = it; 182 } 183 complete_ = complete_session; 184 } 185 } 186 187 void VCMSessionInfo::UpdateDecodableSession(const FrameData& frame_data) { 188 // Irrelevant if session is already complete or decodable 189 if (complete_ || decodable_) 190 return; 191 // TODO(agalusza): Account for bursty loss. 192 // TODO(agalusza): Refine these values to better approximate optimal ones. 193 if (frame_data.rtt_ms < kRttThreshold 194 || frame_type_ == kVideoFrameKey 195 || !HaveFirstPacket() 196 || (NumPackets() <= kHighPacketPercentageThreshold 197 * frame_data.rolling_average_packets_per_frame 198 && NumPackets() > kLowPacketPercentageThreshold 199 * frame_data.rolling_average_packets_per_frame)) 200 return; 201 202 decodable_ = true; 203 } 204 205 bool VCMSessionInfo::complete() const { 206 return complete_; 207 } 208 209 bool VCMSessionInfo::decodable() const { 210 return decodable_; 211 } 212 213 // Find the end of the NAL unit which the packet pointed to by |packet_it| 214 // belongs to. Returns an iterator to the last packet of the frame if the end 215 // of the NAL unit wasn't found. 216 VCMSessionInfo::PacketIterator VCMSessionInfo::FindNaluEnd( 217 PacketIterator packet_it) const { 218 if ((*packet_it).completeNALU == kNaluEnd || 219 (*packet_it).completeNALU == kNaluComplete) { 220 return packet_it; 221 } 222 // Find the end of the NAL unit. 223 for (; packet_it != packets_.end(); ++packet_it) { 224 if (((*packet_it).completeNALU == kNaluComplete && 225 (*packet_it).sizeBytes > 0) || 226 // Found next NALU. 227 (*packet_it).completeNALU == kNaluStart) 228 return --packet_it; 229 if ((*packet_it).completeNALU == kNaluEnd) 230 return packet_it; 231 } 232 // The end wasn't found. 233 return --packet_it; 234 } 235 236 int VCMSessionInfo::DeletePacketData(PacketIterator start, 237 PacketIterator end) { 238 int bytes_to_delete = 0; // The number of bytes to delete. 239 PacketIterator packet_after_end = end; 240 ++packet_after_end; 241 242 // Get the number of bytes to delete. 243 // Clear the size of these packets. 244 for (PacketIterator it = start; it != packet_after_end; ++it) { 245 bytes_to_delete += (*it).sizeBytes; 246 (*it).sizeBytes = 0; 247 (*it).dataPtr = NULL; 248 } 249 if (bytes_to_delete > 0) 250 ShiftSubsequentPackets(end, -bytes_to_delete); 251 return bytes_to_delete; 252 } 253 254 int VCMSessionInfo::BuildVP8FragmentationHeader( 255 uint8_t* frame_buffer, 256 int frame_buffer_length, 257 RTPFragmentationHeader* fragmentation) { 258 int new_length = 0; 259 // Allocate space for max number of partitions 260 fragmentation->VerifyAndAllocateFragmentationHeader(kMaxVP8Partitions); 261 fragmentation->fragmentationVectorSize = 0; 262 memset(fragmentation->fragmentationLength, 0, 263 kMaxVP8Partitions * sizeof(uint32_t)); 264 if (packets_.empty()) 265 return new_length; 266 PacketIterator it = FindNextPartitionBeginning(packets_.begin()); 267 while (it != packets_.end()) { 268 const int partition_id = 269 (*it).codecSpecificHeader.codecHeader.VP8.partitionId; 270 PacketIterator partition_end = FindPartitionEnd(it); 271 fragmentation->fragmentationOffset[partition_id] = 272 (*it).dataPtr - frame_buffer; 273 assert(fragmentation->fragmentationOffset[partition_id] < 274 static_cast<uint32_t>(frame_buffer_length)); 275 fragmentation->fragmentationLength[partition_id] = 276 (*partition_end).dataPtr + (*partition_end).sizeBytes - (*it).dataPtr; 277 assert(fragmentation->fragmentationLength[partition_id] <= 278 static_cast<uint32_t>(frame_buffer_length)); 279 new_length += fragmentation->fragmentationLength[partition_id]; 280 ++partition_end; 281 it = FindNextPartitionBeginning(partition_end); 282 if (partition_id + 1 > fragmentation->fragmentationVectorSize) 283 fragmentation->fragmentationVectorSize = partition_id + 1; 284 } 285 // Set all empty fragments to start where the previous fragment ends, 286 // and have zero length. 287 if (fragmentation->fragmentationLength[0] == 0) 288 fragmentation->fragmentationOffset[0] = 0; 289 for (int i = 1; i < fragmentation->fragmentationVectorSize; ++i) { 290 if (fragmentation->fragmentationLength[i] == 0) 291 fragmentation->fragmentationOffset[i] = 292 fragmentation->fragmentationOffset[i - 1] + 293 fragmentation->fragmentationLength[i - 1]; 294 assert(i == 0 || 295 fragmentation->fragmentationOffset[i] >= 296 fragmentation->fragmentationOffset[i - 1]); 297 } 298 assert(new_length <= frame_buffer_length); 299 return new_length; 300 } 301 302 VCMSessionInfo::PacketIterator VCMSessionInfo::FindNextPartitionBeginning( 303 PacketIterator it) const { 304 while (it != packets_.end()) { 305 if ((*it).codecSpecificHeader.codecHeader.VP8.beginningOfPartition) { 306 return it; 307 } 308 ++it; 309 } 310 return it; 311 } 312 313 VCMSessionInfo::PacketIterator VCMSessionInfo::FindPartitionEnd( 314 PacketIterator it) const { 315 assert((*it).codec == kVideoCodecVP8); 316 PacketIterator prev_it = it; 317 const int partition_id = 318 (*it).codecSpecificHeader.codecHeader.VP8.partitionId; 319 while (it != packets_.end()) { 320 bool beginning = 321 (*it).codecSpecificHeader.codecHeader.VP8.beginningOfPartition; 322 int current_partition_id = 323 (*it).codecSpecificHeader.codecHeader.VP8.partitionId; 324 bool packet_loss_found = (!beginning && !InSequence(it, prev_it)); 325 if (packet_loss_found || 326 (beginning && current_partition_id != partition_id)) { 327 // Missing packet, the previous packet was the last in sequence. 328 return prev_it; 329 } 330 prev_it = it; 331 ++it; 332 } 333 return prev_it; 334 } 335 336 bool VCMSessionInfo::InSequence(const PacketIterator& packet_it, 337 const PacketIterator& prev_packet_it) { 338 // If the two iterators are pointing to the same packet they are considered 339 // to be in sequence. 340 return (packet_it == prev_packet_it || 341 (static_cast<uint16_t>((*prev_packet_it).seqNum + 1) == 342 (*packet_it).seqNum)); 343 } 344 345 int VCMSessionInfo::MakeDecodable() { 346 int return_length = 0; 347 if (packets_.empty()) { 348 return 0; 349 } 350 PacketIterator it = packets_.begin(); 351 // Make sure we remove the first NAL unit if it's not decodable. 352 if ((*it).completeNALU == kNaluIncomplete || 353 (*it).completeNALU == kNaluEnd) { 354 PacketIterator nalu_end = FindNaluEnd(it); 355 return_length += DeletePacketData(it, nalu_end); 356 it = nalu_end; 357 } 358 PacketIterator prev_it = it; 359 // Take care of the rest of the NAL units. 360 for (; it != packets_.end(); ++it) { 361 bool start_of_nalu = ((*it).completeNALU == kNaluStart || 362 (*it).completeNALU == kNaluComplete); 363 if (!start_of_nalu && !InSequence(it, prev_it)) { 364 // Found a sequence number gap due to packet loss. 365 PacketIterator nalu_end = FindNaluEnd(it); 366 return_length += DeletePacketData(it, nalu_end); 367 it = nalu_end; 368 } 369 prev_it = it; 370 } 371 return return_length; 372 } 373 374 void VCMSessionInfo::SetNotDecodableIfIncomplete() { 375 // We don't need to check for completeness first because the two are 376 // orthogonal. If complete_ is true, decodable_ is irrelevant. 377 decodable_ = false; 378 } 379 380 bool 381 VCMSessionInfo::HaveFirstPacket() const { 382 return !packets_.empty() && (first_packet_seq_num_ != -1); 383 } 384 385 bool 386 VCMSessionInfo::HaveLastPacket() const { 387 return !packets_.empty() && (last_packet_seq_num_ != -1); 388 } 389 390 bool 391 VCMSessionInfo::session_nack() const { 392 return session_nack_; 393 } 394 395 int VCMSessionInfo::InsertPacket(const VCMPacket& packet, 396 uint8_t* frame_buffer, 397 VCMDecodeErrorMode decode_error_mode, 398 const FrameData& frame_data) { 399 if (packet.frameType == kFrameEmpty) { 400 // Update sequence number of an empty packet. 401 // Only media packets are inserted into the packet list. 402 InformOfEmptyPacket(packet.seqNum); 403 return 0; 404 } 405 406 if (packets_.size() == kMaxPacketsInSession) { 407 LOG(LS_ERROR) << "Max number of packets per frame has been reached."; 408 return -1; 409 } 410 411 // Find the position of this packet in the packet list in sequence number 412 // order and insert it. Loop over the list in reverse order. 413 ReversePacketIterator rit = packets_.rbegin(); 414 for (; rit != packets_.rend(); ++rit) 415 if (LatestSequenceNumber(packet.seqNum, (*rit).seqNum) == packet.seqNum) 416 break; 417 418 // Check for duplicate packets. 419 if (rit != packets_.rend() && 420 (*rit).seqNum == packet.seqNum && (*rit).sizeBytes > 0) 421 return -2; 422 423 // Only insert media packets between first and last packets (when available). 424 // Placing check here, as to properly account for duplicate packets. 425 // Check if this is first packet (only valid for some codecs) 426 // Should only be set for one packet per session. 427 if (packet.isFirstPacket && first_packet_seq_num_ == -1) { 428 // The first packet in a frame signals the frame type. 429 frame_type_ = packet.frameType; 430 // Store the sequence number for the first packet. 431 first_packet_seq_num_ = static_cast<int>(packet.seqNum); 432 } else if (first_packet_seq_num_ != -1 && 433 !IsNewerSequenceNumber(packet.seqNum, first_packet_seq_num_)) { 434 LOG(LS_WARNING) << "Received packet with a sequence number which is out of" 435 "frame boundaries"; 436 return -3; 437 } else if (frame_type_ == kFrameEmpty && packet.frameType != kFrameEmpty) { 438 // Update the frame type with the type of the first media packet. 439 // TODO(mikhal): Can this trigger? 440 frame_type_ = packet.frameType; 441 } 442 443 // Track the marker bit, should only be set for one packet per session. 444 if (packet.markerBit && last_packet_seq_num_ == -1) { 445 last_packet_seq_num_ = static_cast<int>(packet.seqNum); 446 } else if (last_packet_seq_num_ != -1 && 447 IsNewerSequenceNumber(packet.seqNum, last_packet_seq_num_)) { 448 LOG(LS_WARNING) << "Received packet with a sequence number which is out of" 449 "frame boundaries"; 450 return -3; 451 } 452 453 // The insert operation invalidates the iterator |rit|. 454 PacketIterator packet_list_it = packets_.insert(rit.base(), packet); 455 456 int returnLength = InsertBuffer(frame_buffer, packet_list_it); 457 UpdateCompleteSession(); 458 if (decode_error_mode == kWithErrors) 459 decodable_ = true; 460 else if (decode_error_mode == kSelectiveErrors) 461 UpdateDecodableSession(frame_data); 462 return returnLength; 463 } 464 465 void VCMSessionInfo::InformOfEmptyPacket(uint16_t seq_num) { 466 // Empty packets may be FEC or filler packets. They are sequential and 467 // follow the data packets, therefore, we should only keep track of the high 468 // and low sequence numbers and may assume that the packets in between are 469 // empty packets belonging to the same frame (timestamp). 470 if (empty_seq_num_high_ == -1) 471 empty_seq_num_high_ = seq_num; 472 else 473 empty_seq_num_high_ = LatestSequenceNumber(seq_num, empty_seq_num_high_); 474 if (empty_seq_num_low_ == -1 || IsNewerSequenceNumber(empty_seq_num_low_, 475 seq_num)) 476 empty_seq_num_low_ = seq_num; 477 } 478 479 } // namespace webrtc 480