1 /* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "webrtc/modules/audio_coding/main/acm2/acm_receiver.h" 12 13 #include <stdlib.h> // malloc 14 15 #include <algorithm> // sort 16 #include <vector> 17 18 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" 19 #include "webrtc/common_types.h" 20 #include "webrtc/modules/audio_coding/main/acm2/acm_common_defs.h" 21 #include "webrtc/modules/audio_coding/main/acm2/acm_resampler.h" 22 #include "webrtc/modules/audio_coding/main/acm2/call_statistics.h" 23 #include "webrtc/modules/audio_coding/main/acm2/nack.h" 24 #include "webrtc/modules/audio_coding/neteq/interface/audio_decoder.h" 25 #include "webrtc/modules/audio_coding/neteq/interface/neteq.h" 26 #include "webrtc/system_wrappers/interface/clock.h" 27 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" 28 #include "webrtc/system_wrappers/interface/logging.h" 29 #include "webrtc/system_wrappers/interface/tick_util.h" 30 #include "webrtc/system_wrappers/interface/trace.h" 31 32 namespace webrtc { 33 34 namespace acm2 { 35 36 namespace { 37 38 const int kNackThresholdPackets = 2; 39 40 // |vad_activity_| field of |audio_frame| is set to |previous_audio_activity_| 41 // before the call to this function. 42 void SetAudioFrameActivityAndType(bool vad_enabled, 43 NetEqOutputType type, 44 AudioFrame* audio_frame) { 45 if (vad_enabled) { 46 switch (type) { 47 case kOutputNormal: { 48 audio_frame->vad_activity_ = AudioFrame::kVadActive; 49 audio_frame->speech_type_ = AudioFrame::kNormalSpeech; 50 break; 51 } 52 case kOutputVADPassive: { 53 audio_frame->vad_activity_ = AudioFrame::kVadPassive; 54 audio_frame->speech_type_ = AudioFrame::kNormalSpeech; 55 break; 56 } 57 case kOutputCNG: { 58 audio_frame->vad_activity_ = AudioFrame::kVadPassive; 59 audio_frame->speech_type_ = AudioFrame::kCNG; 60 break; 61 } 62 case kOutputPLC: { 63 // Don't change |audio_frame->vad_activity_|, it should be the same as 64 // |previous_audio_activity_|. 65 audio_frame->speech_type_ = AudioFrame::kPLC; 66 break; 67 } 68 case kOutputPLCtoCNG: { 69 audio_frame->vad_activity_ = AudioFrame::kVadPassive; 70 audio_frame->speech_type_ = AudioFrame::kPLCCNG; 71 break; 72 } 73 default: 74 assert(false); 75 } 76 } else { 77 // Always return kVadUnknown when receive VAD is inactive 78 audio_frame->vad_activity_ = AudioFrame::kVadUnknown; 79 switch (type) { 80 case kOutputNormal: { 81 audio_frame->speech_type_ = AudioFrame::kNormalSpeech; 82 break; 83 } 84 case kOutputCNG: { 85 audio_frame->speech_type_ = AudioFrame::kCNG; 86 break; 87 } 88 case kOutputPLC: { 89 audio_frame->speech_type_ = AudioFrame::kPLC; 90 break; 91 } 92 case kOutputPLCtoCNG: { 93 audio_frame->speech_type_ = AudioFrame::kPLCCNG; 94 break; 95 } 96 case kOutputVADPassive: { 97 // Normally, we should no get any VAD decision if post-decoding VAD is 98 // not active. However, if post-decoding VAD has been active then 99 // disabled, we might be here for couple of frames. 100 audio_frame->speech_type_ = AudioFrame::kNormalSpeech; 101 LOG_F(LS_WARNING) << "Post-decoding VAD is disabled but output is " 102 << "labeled VAD-passive"; 103 break; 104 } 105 default: 106 assert(false); 107 } 108 } 109 } 110 111 // Is the given codec a CNG codec? 112 bool IsCng(int codec_id) { 113 return (codec_id == ACMCodecDB::kCNNB || codec_id == ACMCodecDB::kCNWB || 114 codec_id == ACMCodecDB::kCNSWB || codec_id == ACMCodecDB::kCNFB); 115 } 116 117 } // namespace 118 119 AcmReceiver::AcmReceiver(const AudioCodingModule::Config& config) 120 : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), 121 id_(config.id), 122 last_audio_decoder_(-1), // Invalid value. 123 previous_audio_activity_(AudioFrame::kVadPassive), 124 current_sample_rate_hz_(config.neteq_config.sample_rate_hz), 125 nack_(), 126 nack_enabled_(false), 127 neteq_(NetEq::Create(config.neteq_config)), 128 vad_enabled_(true), 129 clock_(config.clock), 130 av_sync_(false), 131 initial_delay_manager_(), 132 missing_packets_sync_stream_(), 133 late_packets_sync_stream_() { 134 assert(clock_); 135 for (int n = 0; n < ACMCodecDB::kMaxNumCodecs; ++n) { 136 decoders_[n].registered = false; 137 } 138 139 // Make sure we are on the same page as NetEq. Post-decode VAD is disabled by 140 // default in NetEq4, however, Audio Conference Mixer relies on VAD decision 141 // and fails if VAD decision is not provided. 142 if (vad_enabled_) 143 neteq_->EnableVad(); 144 else 145 neteq_->DisableVad(); 146 } 147 148 AcmReceiver::~AcmReceiver() { 149 delete neteq_; 150 } 151 152 int AcmReceiver::SetMinimumDelay(int delay_ms) { 153 if (neteq_->SetMinimumDelay(delay_ms)) 154 return 0; 155 LOG_FERR1(LS_ERROR, "AcmReceiver::SetExtraDelay", delay_ms); 156 return -1; 157 } 158 159 int AcmReceiver::SetInitialDelay(int delay_ms) { 160 if (delay_ms < 0 || delay_ms > 10000) { 161 return -1; 162 } 163 CriticalSectionScoped lock(crit_sect_.get()); 164 165 if (delay_ms == 0) { 166 av_sync_ = false; 167 initial_delay_manager_.reset(); 168 missing_packets_sync_stream_.reset(); 169 late_packets_sync_stream_.reset(); 170 neteq_->SetMinimumDelay(0); 171 return 0; 172 } 173 174 if (av_sync_ && initial_delay_manager_->PacketBuffered()) { 175 // Too late for this API. Only works before a call is started. 176 return -1; 177 } 178 179 // Most of places NetEq calls are not within AcmReceiver's critical section to 180 // improve performance. Here, this call has to be placed before the following 181 // block, therefore, we keep it inside critical section. Otherwise, we have to 182 // release |neteq_crit_sect_| and acquire it again, which seems an overkill. 183 if (!neteq_->SetMinimumDelay(delay_ms)) 184 return -1; 185 186 const int kLatePacketThreshold = 5; 187 av_sync_ = true; 188 initial_delay_manager_.reset(new InitialDelayManager(delay_ms, 189 kLatePacketThreshold)); 190 missing_packets_sync_stream_.reset(new InitialDelayManager::SyncStream); 191 late_packets_sync_stream_.reset(new InitialDelayManager::SyncStream); 192 return 0; 193 } 194 195 int AcmReceiver::SetMaximumDelay(int delay_ms) { 196 if (neteq_->SetMaximumDelay(delay_ms)) 197 return 0; 198 LOG_FERR1(LS_ERROR, "AcmReceiver::SetExtraDelay", delay_ms); 199 return -1; 200 } 201 202 int AcmReceiver::LeastRequiredDelayMs() const { 203 return neteq_->LeastRequiredDelayMs(); 204 } 205 206 int AcmReceiver::current_sample_rate_hz() const { 207 CriticalSectionScoped lock(crit_sect_.get()); 208 return current_sample_rate_hz_; 209 } 210 211 // TODO(turajs): use one set of enumerators, e.g. the one defined in 212 // common_types.h 213 // TODO(henrik.lundin): This method is not used any longer. The call hierarchy 214 // stops in voe::Channel::SetNetEQPlayoutMode(). Remove it. 215 void AcmReceiver::SetPlayoutMode(AudioPlayoutMode mode) { 216 enum NetEqPlayoutMode playout_mode = kPlayoutOn; 217 switch (mode) { 218 case voice: 219 playout_mode = kPlayoutOn; 220 break; 221 case fax: // No change to background noise mode. 222 playout_mode = kPlayoutFax; 223 break; 224 case streaming: 225 playout_mode = kPlayoutStreaming; 226 break; 227 case off: 228 playout_mode = kPlayoutOff; 229 break; 230 } 231 neteq_->SetPlayoutMode(playout_mode); 232 } 233 234 AudioPlayoutMode AcmReceiver::PlayoutMode() const { 235 AudioPlayoutMode acm_mode = voice; 236 NetEqPlayoutMode mode = neteq_->PlayoutMode(); 237 switch (mode) { 238 case kPlayoutOn: 239 acm_mode = voice; 240 break; 241 case kPlayoutOff: 242 acm_mode = off; 243 break; 244 case kPlayoutFax: 245 acm_mode = fax; 246 break; 247 case kPlayoutStreaming: 248 acm_mode = streaming; 249 break; 250 default: 251 assert(false); 252 } 253 return acm_mode; 254 } 255 256 int AcmReceiver::InsertPacket(const WebRtcRTPHeader& rtp_header, 257 const uint8_t* incoming_payload, 258 int length_payload) { 259 uint32_t receive_timestamp = 0; 260 InitialDelayManager::PacketType packet_type = 261 InitialDelayManager::kUndefinedPacket; 262 bool new_codec = false; 263 const RTPHeader* header = &rtp_header.header; // Just a shorthand. 264 265 { 266 CriticalSectionScoped lock(crit_sect_.get()); 267 268 int codec_id = RtpHeaderToCodecIndex(*header, incoming_payload); 269 if (codec_id < 0) { 270 LOG_F(LS_ERROR) << "Payload-type " << header->payloadType 271 << " is not registered."; 272 return -1; 273 } 274 assert(codec_id < ACMCodecDB::kMaxNumCodecs); 275 const int sample_rate_hz = ACMCodecDB::CodecFreq(codec_id); 276 receive_timestamp = NowInTimestamp(sample_rate_hz); 277 278 if (IsCng(codec_id)) { 279 // If this is a CNG while the audio codec is not mono skip pushing in 280 // packets into NetEq. 281 if (last_audio_decoder_ >= 0 && 282 decoders_[last_audio_decoder_].channels > 1) 283 return 0; 284 packet_type = InitialDelayManager::kCngPacket; 285 } else if (codec_id == ACMCodecDB::kAVT) { 286 packet_type = InitialDelayManager::kAvtPacket; 287 } else { 288 if (codec_id != last_audio_decoder_) { 289 // This is either the first audio packet or send codec is changed. 290 // Therefore, either NetEq buffer is empty or will be flushed when this 291 // packet inserted. Note that |last_audio_decoder_| is initialized to 292 // an invalid value (-1), hence, the above condition is true for the 293 // very first audio packet. 294 new_codec = true; 295 296 // Updating NACK'sampling rate is required, either first packet is 297 // received or codec is changed. Furthermore, reset is required if codec 298 // is changed (NetEq flushes its buffer so NACK should reset its list). 299 if (nack_enabled_) { 300 assert(nack_.get()); 301 nack_->Reset(); 302 nack_->UpdateSampleRate(sample_rate_hz); 303 } 304 last_audio_decoder_ = codec_id; 305 } 306 packet_type = InitialDelayManager::kAudioPacket; 307 } 308 309 if (nack_enabled_) { 310 assert(nack_.get()); 311 nack_->UpdateLastReceivedPacket(header->sequenceNumber, 312 header->timestamp); 313 } 314 315 if (av_sync_) { 316 assert(initial_delay_manager_.get()); 317 assert(missing_packets_sync_stream_.get()); 318 // This updates |initial_delay_manager_| and specifies an stream of 319 // sync-packets, if required to be inserted. We insert the sync-packets 320 // when AcmReceiver lock is released and |decoder_lock_| is acquired. 321 initial_delay_manager_->UpdateLastReceivedPacket( 322 rtp_header, receive_timestamp, packet_type, new_codec, sample_rate_hz, 323 missing_packets_sync_stream_.get()); 324 } 325 } // |crit_sect_| is released. 326 327 // If |missing_packets_sync_stream_| is allocated then we are in AV-sync and 328 // we may need to insert sync-packets. We don't check |av_sync_| as we are 329 // outside AcmReceiver's critical section. 330 if (missing_packets_sync_stream_.get()) { 331 InsertStreamOfSyncPackets(missing_packets_sync_stream_.get()); 332 } 333 334 if (neteq_->InsertPacket(rtp_header, incoming_payload, length_payload, 335 receive_timestamp) < 0) { 336 LOG_FERR1(LS_ERROR, "AcmReceiver::InsertPacket", header->payloadType) << 337 " Failed to insert packet"; 338 return -1; 339 } 340 return 0; 341 } 342 343 int AcmReceiver::GetAudio(int desired_freq_hz, AudioFrame* audio_frame) { 344 enum NetEqOutputType type; 345 int16_t* ptr_audio_buffer = audio_frame->data_; 346 int samples_per_channel; 347 int num_channels; 348 bool return_silence = false; 349 350 { 351 // Accessing members, take the lock. 352 CriticalSectionScoped lock(crit_sect_.get()); 353 354 if (av_sync_) { 355 assert(initial_delay_manager_.get()); 356 assert(late_packets_sync_stream_.get()); 357 return_silence = GetSilence(desired_freq_hz, audio_frame); 358 uint32_t timestamp_now = NowInTimestamp(current_sample_rate_hz_); 359 initial_delay_manager_->LatePackets(timestamp_now, 360 late_packets_sync_stream_.get()); 361 } 362 363 if (!return_silence) { 364 // This is our initial guess regarding whether a resampling will be 365 // required. It is based on previous sample rate of netEq. Most often, 366 // this is a correct guess, however, in case that incoming payload changes 367 // the resampling might might be needed. By doing so, we avoid an 368 // unnecessary memcpy(). 369 if (desired_freq_hz != -1 && 370 current_sample_rate_hz_ != desired_freq_hz) { 371 ptr_audio_buffer = audio_buffer_; 372 } 373 } 374 } 375 376 // If |late_packets_sync_stream_| is allocated then we have been in AV-sync 377 // mode and we might have to insert sync-packets. 378 if (late_packets_sync_stream_.get()) { 379 InsertStreamOfSyncPackets(late_packets_sync_stream_.get()); 380 if (return_silence) // Silence generated, don't pull from NetEq. 381 return 0; 382 } 383 384 if (neteq_->GetAudio(AudioFrame::kMaxDataSizeSamples, 385 ptr_audio_buffer, 386 &samples_per_channel, 387 &num_channels, &type) != NetEq::kOK) { 388 LOG_FERR0(LS_ERROR, "AcmReceiver::GetAudio") << "NetEq Failed."; 389 return -1; 390 } 391 392 // Accessing members, take the lock. 393 CriticalSectionScoped lock(crit_sect_.get()); 394 395 // Update NACK. 396 int decoded_sequence_num = 0; 397 uint32_t decoded_timestamp = 0; 398 bool update_nack = nack_enabled_ && // Update NACK only if it is enabled. 399 neteq_->DecodedRtpInfo(&decoded_sequence_num, &decoded_timestamp); 400 if (update_nack) { 401 assert(nack_.get()); 402 nack_->UpdateLastDecodedPacket(decoded_sequence_num, decoded_timestamp); 403 } 404 405 // NetEq always returns 10 ms of audio. 406 current_sample_rate_hz_ = samples_per_channel * 100; 407 408 // Update if resampling is required. 409 bool need_resampling = (desired_freq_hz != -1) && 410 (current_sample_rate_hz_ != desired_freq_hz); 411 412 if (ptr_audio_buffer == audio_buffer_) { 413 // Data is written to local buffer. 414 if (need_resampling) { 415 samples_per_channel = 416 resampler_.Resample10Msec(audio_buffer_, 417 current_sample_rate_hz_, 418 desired_freq_hz, 419 num_channels, 420 AudioFrame::kMaxDataSizeSamples, 421 audio_frame->data_); 422 if (samples_per_channel < 0) { 423 LOG_FERR0(LS_ERROR, "AcmReceiver::GetAudio") << "Resampler Failed."; 424 return -1; 425 } 426 } else { 427 // We might end up here ONLY if codec is changed. 428 memcpy(audio_frame->data_, audio_buffer_, samples_per_channel * 429 num_channels * sizeof(int16_t)); 430 } 431 } else { 432 // Data is written into |audio_frame|. 433 if (need_resampling) { 434 // We might end up here ONLY if codec is changed. 435 samples_per_channel = 436 resampler_.Resample10Msec(audio_frame->data_, 437 current_sample_rate_hz_, 438 desired_freq_hz, 439 num_channels, 440 AudioFrame::kMaxDataSizeSamples, 441 audio_buffer_); 442 if (samples_per_channel < 0) { 443 LOG_FERR0(LS_ERROR, "AcmReceiver::GetAudio") << "Resampler Failed."; 444 return -1; 445 } 446 memcpy(audio_frame->data_, audio_buffer_, samples_per_channel * 447 num_channels * sizeof(int16_t)); 448 } 449 } 450 451 audio_frame->num_channels_ = num_channels; 452 audio_frame->samples_per_channel_ = samples_per_channel; 453 audio_frame->sample_rate_hz_ = samples_per_channel * 100; 454 455 // Should set |vad_activity| before calling SetAudioFrameActivityAndType(). 456 audio_frame->vad_activity_ = previous_audio_activity_; 457 SetAudioFrameActivityAndType(vad_enabled_, type, audio_frame); 458 previous_audio_activity_ = audio_frame->vad_activity_; 459 call_stats_.DecodedByNetEq(audio_frame->speech_type_); 460 461 // Computes the RTP timestamp of the first sample in |audio_frame| from 462 // |GetPlayoutTimestamp|, which is the timestamp of the last sample of 463 // |audio_frame|. 464 uint32_t playout_timestamp = 0; 465 if (GetPlayoutTimestamp(&playout_timestamp)) { 466 audio_frame->timestamp_ = 467 playout_timestamp - audio_frame->samples_per_channel_; 468 } else { 469 // Remain 0 until we have a valid |playout_timestamp|. 470 audio_frame->timestamp_ = 0; 471 } 472 473 return 0; 474 } 475 476 int32_t AcmReceiver::AddCodec(int acm_codec_id, 477 uint8_t payload_type, 478 int channels, 479 AudioDecoder* audio_decoder) { 480 assert(acm_codec_id >= 0 && acm_codec_id < ACMCodecDB::kMaxNumCodecs); 481 NetEqDecoder neteq_decoder = ACMCodecDB::neteq_decoders_[acm_codec_id]; 482 483 // Make sure the right decoder is registered for Opus. 484 if (neteq_decoder == kDecoderOpus && channels == 2) { 485 neteq_decoder = kDecoderOpus_2ch; 486 } 487 488 CriticalSectionScoped lock(crit_sect_.get()); 489 490 // The corresponding NetEq decoder ID. 491 // If this coder has been registered before. 492 if (decoders_[acm_codec_id].registered) { 493 if (decoders_[acm_codec_id].payload_type == payload_type && 494 decoders_[acm_codec_id].channels == channels) { 495 // Re-registering the same codec with the same payload-type. Do nothing 496 // and return. 497 return 0; 498 } 499 500 // Changing the payload-type or number of channels for this codec. 501 // First unregister. Then register with new payload-type/channels. 502 if (neteq_->RemovePayloadType(decoders_[acm_codec_id].payload_type) != 503 NetEq::kOK) { 504 LOG_F(LS_ERROR) << "Cannot remover payload " 505 << decoders_[acm_codec_id].payload_type; 506 return -1; 507 } 508 } 509 510 int ret_val; 511 if (!audio_decoder) { 512 ret_val = neteq_->RegisterPayloadType(neteq_decoder, payload_type); 513 } else { 514 ret_val = neteq_->RegisterExternalDecoder( 515 audio_decoder, neteq_decoder, payload_type); 516 } 517 if (ret_val != NetEq::kOK) { 518 LOG_FERR3(LS_ERROR, "AcmReceiver::AddCodec", acm_codec_id, payload_type, 519 channels); 520 // Registration failed, delete the allocated space and set the pointer to 521 // NULL, for the record. 522 decoders_[acm_codec_id].registered = false; 523 return -1; 524 } 525 526 decoders_[acm_codec_id].registered = true; 527 decoders_[acm_codec_id].payload_type = payload_type; 528 decoders_[acm_codec_id].channels = channels; 529 return 0; 530 } 531 532 void AcmReceiver::EnableVad() { 533 neteq_->EnableVad(); 534 CriticalSectionScoped lock(crit_sect_.get()); 535 vad_enabled_ = true; 536 } 537 538 void AcmReceiver::DisableVad() { 539 neteq_->DisableVad(); 540 CriticalSectionScoped lock(crit_sect_.get()); 541 vad_enabled_ = false; 542 } 543 544 void AcmReceiver::FlushBuffers() { 545 neteq_->FlushBuffers(); 546 } 547 548 // If failed in removing one of the codecs, this method continues to remove as 549 // many as it can. 550 int AcmReceiver::RemoveAllCodecs() { 551 int ret_val = 0; 552 CriticalSectionScoped lock(crit_sect_.get()); 553 for (int n = 0; n < ACMCodecDB::kMaxNumCodecs; ++n) { 554 if (decoders_[n].registered) { 555 if (neteq_->RemovePayloadType(decoders_[n].payload_type) == 0) { 556 decoders_[n].registered = false; 557 } else { 558 LOG_F(LS_ERROR) << "Cannot remove payload " 559 << decoders_[n].payload_type; 560 ret_val = -1; 561 } 562 } 563 } 564 // No codec is registered, invalidate last audio decoder. 565 last_audio_decoder_ = -1; 566 return ret_val; 567 } 568 569 int AcmReceiver::RemoveCodec(uint8_t payload_type) { 570 int codec_index = PayloadType2CodecIndex(payload_type); 571 if (codec_index < 0) { // Such a payload-type is not registered. 572 return 0; 573 } 574 if (neteq_->RemovePayloadType(payload_type) != NetEq::kOK) { 575 LOG_FERR1(LS_ERROR, "AcmReceiver::RemoveCodec", payload_type); 576 return -1; 577 } 578 CriticalSectionScoped lock(crit_sect_.get()); 579 decoders_[codec_index].registered = false; 580 if (last_audio_decoder_ == codec_index) 581 last_audio_decoder_ = -1; // Codec is removed, invalidate last decoder. 582 return 0; 583 } 584 585 void AcmReceiver::set_id(int id) { 586 CriticalSectionScoped lock(crit_sect_.get()); 587 id_ = id; 588 } 589 590 bool AcmReceiver::GetPlayoutTimestamp(uint32_t* timestamp) { 591 if (av_sync_) { 592 assert(initial_delay_manager_.get()); 593 if (initial_delay_manager_->buffering()) { 594 return initial_delay_manager_->GetPlayoutTimestamp(timestamp); 595 } 596 } 597 return neteq_->GetPlayoutTimestamp(timestamp); 598 } 599 600 int AcmReceiver::last_audio_codec_id() const { 601 CriticalSectionScoped lock(crit_sect_.get()); 602 return last_audio_decoder_; 603 } 604 605 int AcmReceiver::last_audio_payload_type() const { 606 CriticalSectionScoped lock(crit_sect_.get()); 607 if (last_audio_decoder_ < 0) 608 return -1; 609 assert(decoders_[last_audio_decoder_].registered); 610 return decoders_[last_audio_decoder_].payload_type; 611 } 612 613 int AcmReceiver::RedPayloadType() const { 614 CriticalSectionScoped lock(crit_sect_.get()); 615 if (ACMCodecDB::kRED < 0 || 616 !decoders_[ACMCodecDB::kRED].registered) { 617 LOG_F(LS_WARNING) << "RED is not registered."; 618 return -1; 619 } 620 return decoders_[ACMCodecDB::kRED].payload_type; 621 } 622 623 int AcmReceiver::LastAudioCodec(CodecInst* codec) const { 624 CriticalSectionScoped lock(crit_sect_.get()); 625 if (last_audio_decoder_ < 0) { 626 return -1; 627 } 628 assert(decoders_[last_audio_decoder_].registered); 629 memcpy(codec, &ACMCodecDB::database_[last_audio_decoder_], sizeof(CodecInst)); 630 codec->pltype = decoders_[last_audio_decoder_].payload_type; 631 codec->channels = decoders_[last_audio_decoder_].channels; 632 return 0; 633 } 634 635 void AcmReceiver::NetworkStatistics(ACMNetworkStatistics* acm_stat) { 636 NetEqNetworkStatistics neteq_stat; 637 // NetEq function always returns zero, so we don't check the return value. 638 neteq_->NetworkStatistics(&neteq_stat); 639 640 acm_stat->currentBufferSize = neteq_stat.current_buffer_size_ms; 641 acm_stat->preferredBufferSize = neteq_stat.preferred_buffer_size_ms; 642 acm_stat->jitterPeaksFound = neteq_stat.jitter_peaks_found ? true : false; 643 acm_stat->currentPacketLossRate = neteq_stat.packet_loss_rate; 644 acm_stat->currentDiscardRate = neteq_stat.packet_discard_rate; 645 acm_stat->currentExpandRate = neteq_stat.expand_rate; 646 acm_stat->currentPreemptiveRate = neteq_stat.preemptive_rate; 647 acm_stat->currentAccelerateRate = neteq_stat.accelerate_rate; 648 acm_stat->clockDriftPPM = neteq_stat.clockdrift_ppm; 649 acm_stat->addedSamples = neteq_stat.added_zero_samples; 650 651 std::vector<int> waiting_times; 652 neteq_->WaitingTimes(&waiting_times); 653 size_t size = waiting_times.size(); 654 if (size == 0) { 655 acm_stat->meanWaitingTimeMs = -1; 656 acm_stat->medianWaitingTimeMs = -1; 657 acm_stat->minWaitingTimeMs = -1; 658 acm_stat->maxWaitingTimeMs = -1; 659 } else { 660 std::sort(waiting_times.begin(), waiting_times.end()); 661 if ((size & 0x1) == 0) { 662 acm_stat->medianWaitingTimeMs = (waiting_times[size / 2 - 1] + 663 waiting_times[size / 2]) / 2; 664 } else { 665 acm_stat->medianWaitingTimeMs = waiting_times[size / 2]; 666 } 667 acm_stat->minWaitingTimeMs = waiting_times.front(); 668 acm_stat->maxWaitingTimeMs = waiting_times.back(); 669 double sum = 0; 670 for (size_t i = 0; i < size; ++i) { 671 sum += waiting_times[i]; 672 } 673 acm_stat->meanWaitingTimeMs = static_cast<int>(sum / size); 674 } 675 } 676 677 int AcmReceiver::DecoderByPayloadType(uint8_t payload_type, 678 CodecInst* codec) const { 679 CriticalSectionScoped lock(crit_sect_.get()); 680 int codec_index = PayloadType2CodecIndex(payload_type); 681 if (codec_index < 0) { 682 LOG_FERR1(LS_ERROR, "AcmReceiver::DecoderByPayloadType", payload_type); 683 return -1; 684 } 685 memcpy(codec, &ACMCodecDB::database_[codec_index], sizeof(CodecInst)); 686 codec->pltype = decoders_[codec_index].payload_type; 687 codec->channels = decoders_[codec_index].channels; 688 return 0; 689 } 690 691 int AcmReceiver::PayloadType2CodecIndex(uint8_t payload_type) const { 692 for (int n = 0; n < ACMCodecDB::kMaxNumCodecs; ++n) { 693 if (decoders_[n].registered && decoders_[n].payload_type == payload_type) { 694 return n; 695 } 696 } 697 return -1; 698 } 699 700 int AcmReceiver::EnableNack(size_t max_nack_list_size) { 701 // Don't do anything if |max_nack_list_size| is out of range. 702 if (max_nack_list_size == 0 || max_nack_list_size > Nack::kNackListSizeLimit) 703 return -1; 704 705 CriticalSectionScoped lock(crit_sect_.get()); 706 if (!nack_enabled_) { 707 nack_.reset(Nack::Create(kNackThresholdPackets)); 708 nack_enabled_ = true; 709 710 // Sampling rate might need to be updated if we change from disable to 711 // enable. Do it if the receive codec is valid. 712 if (last_audio_decoder_ >= 0) { 713 nack_->UpdateSampleRate( 714 ACMCodecDB::database_[last_audio_decoder_].plfreq); 715 } 716 } 717 return nack_->SetMaxNackListSize(max_nack_list_size); 718 } 719 720 void AcmReceiver::DisableNack() { 721 CriticalSectionScoped lock(crit_sect_.get()); 722 nack_.reset(); // Memory is released. 723 nack_enabled_ = false; 724 } 725 726 std::vector<uint16_t> AcmReceiver::GetNackList( 727 int round_trip_time_ms) const { 728 CriticalSectionScoped lock(crit_sect_.get()); 729 if (round_trip_time_ms < 0) { 730 WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceAudioCoding, id_, 731 "GetNackList: round trip time cannot be negative." 732 " round_trip_time_ms=%d", round_trip_time_ms); 733 } 734 if (nack_enabled_ && round_trip_time_ms >= 0) { 735 assert(nack_.get()); 736 return nack_->GetNackList(round_trip_time_ms); 737 } 738 std::vector<uint16_t> empty_list; 739 return empty_list; 740 } 741 742 void AcmReceiver::ResetInitialDelay() { 743 { 744 CriticalSectionScoped lock(crit_sect_.get()); 745 av_sync_ = false; 746 initial_delay_manager_.reset(NULL); 747 missing_packets_sync_stream_.reset(NULL); 748 late_packets_sync_stream_.reset(NULL); 749 } 750 neteq_->SetMinimumDelay(0); 751 // TODO(turajs): Should NetEq Buffer be flushed? 752 } 753 754 // This function is called within critical section, no need to acquire a lock. 755 bool AcmReceiver::GetSilence(int desired_sample_rate_hz, AudioFrame* frame) { 756 assert(av_sync_); 757 assert(initial_delay_manager_.get()); 758 if (!initial_delay_manager_->buffering()) { 759 return false; 760 } 761 762 // We stop accumulating packets, if the number of packets or the total size 763 // exceeds a threshold. 764 int num_packets; 765 int max_num_packets; 766 const float kBufferingThresholdScale = 0.9f; 767 neteq_->PacketBufferStatistics(&num_packets, &max_num_packets); 768 if (num_packets > max_num_packets * kBufferingThresholdScale) { 769 initial_delay_manager_->DisableBuffering(); 770 return false; 771 } 772 773 // Update statistics. 774 call_stats_.DecodedBySilenceGenerator(); 775 776 // Set the values if already got a packet, otherwise set to default values. 777 if (last_audio_decoder_ >= 0) { 778 current_sample_rate_hz_ = ACMCodecDB::database_[last_audio_decoder_].plfreq; 779 frame->num_channels_ = decoders_[last_audio_decoder_].channels; 780 } else { 781 frame->num_channels_ = 1; 782 } 783 784 // Set the audio frame's sampling frequency. 785 if (desired_sample_rate_hz > 0) { 786 frame->sample_rate_hz_ = desired_sample_rate_hz; 787 } else { 788 frame->sample_rate_hz_ = current_sample_rate_hz_; 789 } 790 791 frame->samples_per_channel_ = frame->sample_rate_hz_ / 100; // Always 10 ms. 792 frame->speech_type_ = AudioFrame::kCNG; 793 frame->vad_activity_ = AudioFrame::kVadPassive; 794 int samples = frame->samples_per_channel_ * frame->num_channels_; 795 memset(frame->data_, 0, samples * sizeof(int16_t)); 796 return true; 797 } 798 799 int AcmReceiver::RtpHeaderToCodecIndex( 800 const RTPHeader &rtp_header, const uint8_t* payload) const { 801 uint8_t payload_type = rtp_header.payloadType; 802 if (ACMCodecDB::kRED >= 0 && // This ensures that RED is defined in WebRTC. 803 decoders_[ACMCodecDB::kRED].registered && 804 payload_type == decoders_[ACMCodecDB::kRED].payload_type) { 805 // This is a RED packet, get the payload of the audio codec. 806 payload_type = payload[0] & 0x7F; 807 } 808 809 // Check if the payload is registered. 810 return PayloadType2CodecIndex(payload_type); 811 } 812 813 uint32_t AcmReceiver::NowInTimestamp(int decoder_sampling_rate) const { 814 // Down-cast the time to (32-6)-bit since we only care about 815 // the least significant bits. (32-6) bits cover 2^(32-6) = 67108864 ms. 816 // We masked 6 most significant bits of 32-bit so there is no overflow in 817 // the conversion from milliseconds to timestamp. 818 const uint32_t now_in_ms = static_cast<uint32_t>( 819 clock_->TimeInMilliseconds() & 0x03ffffff); 820 return static_cast<uint32_t>( 821 (decoder_sampling_rate / 1000) * now_in_ms); 822 } 823 824 // This function only interacts with |neteq_|, therefore, it does not have to 825 // be within critical section of AcmReceiver. It is inserting packets 826 // into NetEq, so we call it when |decode_lock_| is acquired. However, this is 827 // not essential as sync-packets do not interact with codecs (especially BWE). 828 void AcmReceiver::InsertStreamOfSyncPackets( 829 InitialDelayManager::SyncStream* sync_stream) { 830 assert(sync_stream); 831 assert(av_sync_); 832 for (int n = 0; n < sync_stream->num_sync_packets; ++n) { 833 neteq_->InsertSyncPacket(sync_stream->rtp_info, 834 sync_stream->receive_timestamp); 835 ++sync_stream->rtp_info.header.sequenceNumber; 836 sync_stream->rtp_info.header.timestamp += sync_stream->timestamp_step; 837 sync_stream->receive_timestamp += sync_stream->timestamp_step; 838 } 839 } 840 841 void AcmReceiver::GetDecodingCallStatistics( 842 AudioDecodingCallStats* stats) const { 843 CriticalSectionScoped lock(crit_sect_.get()); 844 *stats = call_stats_.GetDecodingStatistics(); 845 } 846 847 } // namespace acm2 848 849 } // namespace webrtc 850
