1 /* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "webrtc/modules/audio_coding/main/acm2/acm_receiver.h" 12 13 #include <stdlib.h> // malloc 14 15 #include <algorithm> // sort 16 #include <vector> 17 18 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" 19 #include "webrtc/common_types.h" 20 #include "webrtc/modules/audio_coding/main/acm2/acm_common_defs.h" 21 #include "webrtc/modules/audio_coding/main/acm2/acm_resampler.h" 22 #include "webrtc/modules/audio_coding/main/acm2/call_statistics.h" 23 #include "webrtc/modules/audio_coding/main/acm2/nack.h" 24 #include "webrtc/modules/audio_coding/neteq/interface/audio_decoder.h" 25 #include "webrtc/modules/audio_coding/neteq/interface/neteq.h" 26 #include "webrtc/system_wrappers/interface/clock.h" 27 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" 28 #include "webrtc/system_wrappers/interface/logging.h" 29 #include "webrtc/system_wrappers/interface/rw_lock_wrapper.h" 30 #include "webrtc/system_wrappers/interface/tick_util.h" 31 #include "webrtc/system_wrappers/interface/trace.h" 32 33 namespace webrtc { 34 35 namespace acm2 { 36 37 namespace { 38 39 const int kNackThresholdPackets = 2; 40 41 // |vad_activity_| field of |audio_frame| is set to |previous_audio_activity_| 42 // before the call to this function. 43 void SetAudioFrameActivityAndType(bool vad_enabled, 44 NetEqOutputType type, 45 AudioFrame* audio_frame) { 46 if (vad_enabled) { 47 switch (type) { 48 case kOutputNormal: { 49 audio_frame->vad_activity_ = AudioFrame::kVadActive; 50 audio_frame->speech_type_ = AudioFrame::kNormalSpeech; 51 break; 52 } 53 case kOutputVADPassive: { 54 audio_frame->vad_activity_ = AudioFrame::kVadPassive; 55 audio_frame->speech_type_ = AudioFrame::kNormalSpeech; 56 break; 57 } 58 case kOutputCNG: { 59 audio_frame->vad_activity_ = AudioFrame::kVadPassive; 60 audio_frame->speech_type_ = AudioFrame::kCNG; 61 break; 62 } 63 case kOutputPLC: { 64 // Don't change |audio_frame->vad_activity_|, it should be the same as 65 // |previous_audio_activity_|. 66 audio_frame->speech_type_ = AudioFrame::kPLC; 67 break; 68 } 69 case kOutputPLCtoCNG: { 70 audio_frame->vad_activity_ = AudioFrame::kVadPassive; 71 audio_frame->speech_type_ = AudioFrame::kPLCCNG; 72 break; 73 } 74 default: 75 assert(false); 76 } 77 } else { 78 // Always return kVadUnknown when receive VAD is inactive 79 audio_frame->vad_activity_ = AudioFrame::kVadUnknown; 80 switch (type) { 81 case kOutputNormal: { 82 audio_frame->speech_type_ = AudioFrame::kNormalSpeech; 83 break; 84 } 85 case kOutputCNG: { 86 audio_frame->speech_type_ = AudioFrame::kCNG; 87 break; 88 } 89 case kOutputPLC: { 90 audio_frame->speech_type_ = AudioFrame::kPLC; 91 break; 92 } 93 case kOutputPLCtoCNG: { 94 audio_frame->speech_type_ = AudioFrame::kPLCCNG; 95 break; 96 } 97 case kOutputVADPassive: { 98 // Normally, we should no get any VAD decision if post-decoding VAD is 99 // not active. However, if post-decoding VAD has been active then 100 // disabled, we might be here for couple of frames. 101 audio_frame->speech_type_ = AudioFrame::kNormalSpeech; 102 LOG_F(LS_WARNING) << "Post-decoding VAD is disabled but output is " 103 << "labeled VAD-passive"; 104 break; 105 } 106 default: 107 assert(false); 108 } 109 } 110 } 111 112 // Is the given codec a CNG codec? 113 bool IsCng(int codec_id) { 114 return (codec_id == ACMCodecDB::kCNNB || codec_id == ACMCodecDB::kCNWB || 115 codec_id == ACMCodecDB::kCNSWB || codec_id == ACMCodecDB::kCNFB); 116 } 117 118 } // namespace 119 120 AcmReceiver::AcmReceiver(const AudioCodingModule::Config& config) 121 : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), 122 id_(config.id), 123 last_audio_decoder_(-1), // Invalid value. 124 previous_audio_activity_(AudioFrame::kVadPassive), 125 current_sample_rate_hz_(config.neteq_config.sample_rate_hz), 126 nack_(), 127 nack_enabled_(false), 128 neteq_(NetEq::Create(config.neteq_config)), 129 decode_lock_(RWLockWrapper::CreateRWLock()), 130 vad_enabled_(true), 131 clock_(config.clock), 132 av_sync_(false), 133 initial_delay_manager_(), 134 missing_packets_sync_stream_(), 135 late_packets_sync_stream_() { 136 assert(clock_); 137 for (int n = 0; n < ACMCodecDB::kMaxNumCodecs; ++n) { 138 decoders_[n].registered = false; 139 } 140 141 // Make sure we are on the same page as NetEq. Post-decode VAD is disabled by 142 // default in NetEq4, however, Audio Conference Mixer relies on VAD decision 143 // and fails if VAD decision is not provided. 144 if (vad_enabled_) 145 neteq_->EnableVad(); 146 else 147 neteq_->DisableVad(); 148 } 149 150 AcmReceiver::~AcmReceiver() { 151 delete neteq_; 152 delete decode_lock_; 153 } 154 155 int AcmReceiver::SetMinimumDelay(int delay_ms) { 156 if (neteq_->SetMinimumDelay(delay_ms)) 157 return 0; 158 LOG_FERR1(LS_ERROR, "AcmReceiver::SetExtraDelay", delay_ms); 159 return -1; 160 } 161 162 int AcmReceiver::SetInitialDelay(int delay_ms) { 163 if (delay_ms < 0 || delay_ms > 10000) { 164 return -1; 165 } 166 CriticalSectionScoped lock(crit_sect_.get()); 167 168 if (delay_ms == 0) { 169 av_sync_ = false; 170 initial_delay_manager_.reset(); 171 missing_packets_sync_stream_.reset(); 172 late_packets_sync_stream_.reset(); 173 neteq_->SetMinimumDelay(0); 174 return 0; 175 } 176 177 if (av_sync_ && initial_delay_manager_->PacketBuffered()) { 178 // Too late for this API. Only works before a call is started. 179 return -1; 180 } 181 182 // Most of places NetEq calls are not within AcmReceiver's critical section to 183 // improve performance. Here, this call has to be placed before the following 184 // block, therefore, we keep it inside critical section. Otherwise, we have to 185 // release |neteq_crit_sect_| and acquire it again, which seems an overkill. 186 if (!neteq_->SetMinimumDelay(delay_ms)) 187 return -1; 188 189 const int kLatePacketThreshold = 5; 190 av_sync_ = true; 191 initial_delay_manager_.reset(new InitialDelayManager(delay_ms, 192 kLatePacketThreshold)); 193 missing_packets_sync_stream_.reset(new InitialDelayManager::SyncStream); 194 late_packets_sync_stream_.reset(new InitialDelayManager::SyncStream); 195 return 0; 196 } 197 198 int AcmReceiver::SetMaximumDelay(int delay_ms) { 199 if (neteq_->SetMaximumDelay(delay_ms)) 200 return 0; 201 LOG_FERR1(LS_ERROR, "AcmReceiver::SetExtraDelay", delay_ms); 202 return -1; 203 } 204 205 int AcmReceiver::LeastRequiredDelayMs() const { 206 return neteq_->LeastRequiredDelayMs(); 207 } 208 209 int AcmReceiver::current_sample_rate_hz() const { 210 CriticalSectionScoped lock(crit_sect_.get()); 211 return current_sample_rate_hz_; 212 } 213 214 // TODO(turajs): use one set of enumerators, e.g. the one defined in 215 // common_types.h 216 void AcmReceiver::SetPlayoutMode(AudioPlayoutMode mode) { 217 enum NetEqPlayoutMode playout_mode = kPlayoutOn; 218 enum NetEqBackgroundNoiseMode bgn_mode = kBgnOn; 219 switch (mode) { 220 case voice: 221 playout_mode = kPlayoutOn; 222 bgn_mode = kBgnOn; 223 break; 224 case fax: // No change to background noise mode. 225 playout_mode = kPlayoutFax; 226 bgn_mode = neteq_->BackgroundNoiseMode(); 227 break; 228 case streaming: 229 playout_mode = kPlayoutStreaming; 230 bgn_mode = kBgnOff; 231 break; 232 case off: 233 playout_mode = kPlayoutOff; 234 bgn_mode = kBgnOff; 235 break; 236 } 237 neteq_->SetPlayoutMode(playout_mode); 238 neteq_->SetBackgroundNoiseMode(bgn_mode); 239 } 240 241 AudioPlayoutMode AcmReceiver::PlayoutMode() const { 242 AudioPlayoutMode acm_mode = voice; 243 NetEqPlayoutMode mode = neteq_->PlayoutMode(); 244 switch (mode) { 245 case kPlayoutOn: 246 acm_mode = voice; 247 break; 248 case kPlayoutOff: 249 acm_mode = off; 250 break; 251 case kPlayoutFax: 252 acm_mode = fax; 253 break; 254 case kPlayoutStreaming: 255 acm_mode = streaming; 256 break; 257 default: 258 assert(false); 259 } 260 return acm_mode; 261 } 262 263 int AcmReceiver::InsertPacket(const WebRtcRTPHeader& rtp_header, 264 const uint8_t* incoming_payload, 265 int length_payload) { 266 uint32_t receive_timestamp = 0; 267 InitialDelayManager::PacketType packet_type = 268 InitialDelayManager::kUndefinedPacket; 269 bool new_codec = false; 270 const RTPHeader* header = &rtp_header.header; // Just a shorthand. 271 272 { 273 CriticalSectionScoped lock(crit_sect_.get()); 274 275 int codec_id = RtpHeaderToCodecIndex(*header, incoming_payload); 276 if (codec_id < 0) { 277 LOG_F(LS_ERROR) << "Payload-type " << header->payloadType 278 << " is not registered."; 279 return -1; 280 } 281 assert(codec_id < ACMCodecDB::kMaxNumCodecs); 282 const int sample_rate_hz = ACMCodecDB::CodecFreq(codec_id); 283 receive_timestamp = NowInTimestamp(sample_rate_hz); 284 285 if (IsCng(codec_id)) { 286 // If this is a CNG while the audio codec is not mono skip pushing in 287 // packets into NetEq. 288 if (last_audio_decoder_ >= 0 && 289 decoders_[last_audio_decoder_].channels > 1) 290 return 0; 291 packet_type = InitialDelayManager::kCngPacket; 292 } else if (codec_id == ACMCodecDB::kAVT) { 293 packet_type = InitialDelayManager::kAvtPacket; 294 } else { 295 if (codec_id != last_audio_decoder_) { 296 // This is either the first audio packet or send codec is changed. 297 // Therefore, either NetEq buffer is empty or will be flushed when this 298 // packet inserted. Note that |last_audio_decoder_| is initialized to 299 // an invalid value (-1), hence, the above condition is true for the 300 // very first audio packet. 301 new_codec = true; 302 303 // Updating NACK'sampling rate is required, either first packet is 304 // received or codec is changed. Furthermore, reset is required if codec 305 // is changed (NetEq flushes its buffer so NACK should reset its list). 306 if (nack_enabled_) { 307 assert(nack_.get()); 308 nack_->Reset(); 309 nack_->UpdateSampleRate(sample_rate_hz); 310 } 311 last_audio_decoder_ = codec_id; 312 } 313 packet_type = InitialDelayManager::kAudioPacket; 314 } 315 316 if (nack_enabled_) { 317 assert(nack_.get()); 318 nack_->UpdateLastReceivedPacket(header->sequenceNumber, 319 header->timestamp); 320 } 321 322 if (av_sync_) { 323 assert(initial_delay_manager_.get()); 324 assert(missing_packets_sync_stream_.get()); 325 // This updates |initial_delay_manager_| and specifies an stream of 326 // sync-packets, if required to be inserted. We insert the sync-packets 327 // when AcmReceiver lock is released and |decoder_lock_| is acquired. 328 initial_delay_manager_->UpdateLastReceivedPacket( 329 rtp_header, receive_timestamp, packet_type, new_codec, sample_rate_hz, 330 missing_packets_sync_stream_.get()); 331 } 332 } // |crit_sect_| is released. 333 334 { 335 WriteLockScoped lock_codecs(*decode_lock_); // Lock to prevent an encoding. 336 337 // If |missing_packets_sync_stream_| is allocated then we are in AV-sync and 338 // we may need to insert sync-packets. We don't check |av_sync_| as we are 339 // outside AcmReceiver's critical section. 340 if (missing_packets_sync_stream_.get()) { 341 InsertStreamOfSyncPackets(missing_packets_sync_stream_.get()); 342 } 343 344 if (neteq_->InsertPacket(rtp_header, incoming_payload, length_payload, 345 receive_timestamp) < 0) { 346 LOG_FERR1(LS_ERROR, "AcmReceiver::InsertPacket", header->payloadType) << 347 " Failed to insert packet"; 348 return -1; 349 } 350 } 351 return 0; 352 } 353 354 int AcmReceiver::GetAudio(int desired_freq_hz, AudioFrame* audio_frame) { 355 enum NetEqOutputType type; 356 int16_t* ptr_audio_buffer = audio_frame->data_; 357 int samples_per_channel; 358 int num_channels; 359 bool return_silence = false; 360 361 { 362 // Accessing members, take the lock. 363 CriticalSectionScoped lock(crit_sect_.get()); 364 365 if (av_sync_) { 366 assert(initial_delay_manager_.get()); 367 assert(late_packets_sync_stream_.get()); 368 return_silence = GetSilence(desired_freq_hz, audio_frame); 369 uint32_t timestamp_now = NowInTimestamp(current_sample_rate_hz_); 370 initial_delay_manager_->LatePackets(timestamp_now, 371 late_packets_sync_stream_.get()); 372 } 373 374 if (!return_silence) { 375 // This is our initial guess regarding whether a resampling will be 376 // required. It is based on previous sample rate of netEq. Most often, 377 // this is a correct guess, however, in case that incoming payload changes 378 // the resampling might might be needed. By doing so, we avoid an 379 // unnecessary memcpy(). 380 if (desired_freq_hz != -1 && 381 current_sample_rate_hz_ != desired_freq_hz) { 382 ptr_audio_buffer = audio_buffer_; 383 } 384 } 385 } 386 387 { 388 WriteLockScoped lock_codecs(*decode_lock_); // Lock to prevent an encoding. 389 390 // If |late_packets_sync_stream_| is allocated then we have been in AV-sync 391 // mode and we might have to insert sync-packets. 392 if (late_packets_sync_stream_.get()) { 393 InsertStreamOfSyncPackets(late_packets_sync_stream_.get()); 394 if (return_silence) // Silence generated, don't pull from NetEq. 395 return 0; 396 } 397 398 if (neteq_->GetAudio(AudioFrame::kMaxDataSizeSamples, 399 ptr_audio_buffer, 400 &samples_per_channel, 401 &num_channels, &type) != NetEq::kOK) { 402 LOG_FERR0(LS_ERROR, "AcmReceiver::GetAudio") << "NetEq Failed."; 403 return -1; 404 } 405 } 406 407 // Accessing members, take the lock. 408 CriticalSectionScoped lock(crit_sect_.get()); 409 410 // Update NACK. 411 int decoded_sequence_num = 0; 412 uint32_t decoded_timestamp = 0; 413 bool update_nack = nack_enabled_ && // Update NACK only if it is enabled. 414 neteq_->DecodedRtpInfo(&decoded_sequence_num, &decoded_timestamp); 415 if (update_nack) { 416 assert(nack_.get()); 417 nack_->UpdateLastDecodedPacket(decoded_sequence_num, decoded_timestamp); 418 } 419 420 // NetEq always returns 10 ms of audio. 421 current_sample_rate_hz_ = samples_per_channel * 100; 422 423 // Update if resampling is required. 424 bool need_resampling = (desired_freq_hz != -1) && 425 (current_sample_rate_hz_ != desired_freq_hz); 426 427 if (ptr_audio_buffer == audio_buffer_) { 428 // Data is written to local buffer. 429 if (need_resampling) { 430 samples_per_channel = 431 resampler_.Resample10Msec(audio_buffer_, 432 current_sample_rate_hz_, 433 desired_freq_hz, 434 num_channels, 435 AudioFrame::kMaxDataSizeSamples, 436 audio_frame->data_); 437 if (samples_per_channel < 0) { 438 LOG_FERR0(LS_ERROR, "AcmReceiver::GetAudio") << "Resampler Failed."; 439 return -1; 440 } 441 } else { 442 // We might end up here ONLY if codec is changed. 443 memcpy(audio_frame->data_, audio_buffer_, samples_per_channel * 444 num_channels * sizeof(int16_t)); 445 } 446 } else { 447 // Data is written into |audio_frame|. 448 if (need_resampling) { 449 // We might end up here ONLY if codec is changed. 450 samples_per_channel = 451 resampler_.Resample10Msec(audio_frame->data_, 452 current_sample_rate_hz_, 453 desired_freq_hz, 454 num_channels, 455 AudioFrame::kMaxDataSizeSamples, 456 audio_buffer_); 457 if (samples_per_channel < 0) { 458 LOG_FERR0(LS_ERROR, "AcmReceiver::GetAudio") << "Resampler Failed."; 459 return -1; 460 } 461 memcpy(audio_frame->data_, audio_buffer_, samples_per_channel * 462 num_channels * sizeof(int16_t)); 463 } 464 } 465 466 audio_frame->num_channels_ = num_channels; 467 audio_frame->samples_per_channel_ = samples_per_channel; 468 audio_frame->sample_rate_hz_ = samples_per_channel * 100; 469 470 // Should set |vad_activity| before calling SetAudioFrameActivityAndType(). 471 audio_frame->vad_activity_ = previous_audio_activity_; 472 SetAudioFrameActivityAndType(vad_enabled_, type, audio_frame); 473 previous_audio_activity_ = audio_frame->vad_activity_; 474 call_stats_.DecodedByNetEq(audio_frame->speech_type_); 475 476 // Computes the RTP timestamp of the first sample in |audio_frame| from 477 // |GetPlayoutTimestamp|, which is the timestamp of the last sample of 478 // |audio_frame|. 479 uint32_t playout_timestamp = 0; 480 if (GetPlayoutTimestamp(&playout_timestamp)) { 481 audio_frame->timestamp_ = 482 playout_timestamp - audio_frame->samples_per_channel_; 483 } else { 484 // Remain 0 until we have a valid |playout_timestamp|. 485 audio_frame->timestamp_ = 0; 486 } 487 488 return 0; 489 } 490 491 int32_t AcmReceiver::AddCodec(int acm_codec_id, 492 uint8_t payload_type, 493 int channels, 494 AudioDecoder* audio_decoder) { 495 assert(acm_codec_id >= 0 && acm_codec_id < ACMCodecDB::kMaxNumCodecs); 496 NetEqDecoder neteq_decoder = ACMCodecDB::neteq_decoders_[acm_codec_id]; 497 498 // Make sure the right decoder is registered for Opus. 499 if (neteq_decoder == kDecoderOpus && channels == 2) { 500 neteq_decoder = kDecoderOpus_2ch; 501 } 502 503 CriticalSectionScoped lock(crit_sect_.get()); 504 505 // The corresponding NetEq decoder ID. 506 // If this coder has been registered before. 507 if (decoders_[acm_codec_id].registered) { 508 if (decoders_[acm_codec_id].payload_type == payload_type && 509 decoders_[acm_codec_id].channels == channels) { 510 // Re-registering the same codec with the same payload-type. Do nothing 511 // and return. 512 return 0; 513 } 514 515 // Changing the payload-type or number of channels for this codec. 516 // First unregister. Then register with new payload-type/channels. 517 if (neteq_->RemovePayloadType(decoders_[acm_codec_id].payload_type) != 518 NetEq::kOK) { 519 LOG_F(LS_ERROR) << "Cannot remover payload " 520 << decoders_[acm_codec_id].payload_type; 521 return -1; 522 } 523 } 524 525 int ret_val; 526 if (!audio_decoder) { 527 ret_val = neteq_->RegisterPayloadType(neteq_decoder, payload_type); 528 } else { 529 ret_val = neteq_->RegisterExternalDecoder( 530 audio_decoder, neteq_decoder, payload_type); 531 } 532 if (ret_val != NetEq::kOK) { 533 LOG_FERR3(LS_ERROR, "AcmReceiver::AddCodec", acm_codec_id, payload_type, 534 channels); 535 // Registration failed, delete the allocated space and set the pointer to 536 // NULL, for the record. 537 decoders_[acm_codec_id].registered = false; 538 return -1; 539 } 540 541 decoders_[acm_codec_id].registered = true; 542 decoders_[acm_codec_id].payload_type = payload_type; 543 decoders_[acm_codec_id].channels = channels; 544 return 0; 545 } 546 547 void AcmReceiver::EnableVad() { 548 neteq_->EnableVad(); 549 CriticalSectionScoped lock(crit_sect_.get()); 550 vad_enabled_ = true; 551 } 552 553 void AcmReceiver::DisableVad() { 554 neteq_->DisableVad(); 555 CriticalSectionScoped lock(crit_sect_.get()); 556 vad_enabled_ = false; 557 } 558 559 void AcmReceiver::FlushBuffers() { 560 neteq_->FlushBuffers(); 561 } 562 563 // If failed in removing one of the codecs, this method continues to remove as 564 // many as it can. 565 int AcmReceiver::RemoveAllCodecs() { 566 int ret_val = 0; 567 CriticalSectionScoped lock(crit_sect_.get()); 568 for (int n = 0; n < ACMCodecDB::kMaxNumCodecs; ++n) { 569 if (decoders_[n].registered) { 570 if (neteq_->RemovePayloadType(decoders_[n].payload_type) == 0) { 571 decoders_[n].registered = false; 572 } else { 573 LOG_F(LS_ERROR) << "Cannot remove payload " 574 << decoders_[n].payload_type; 575 ret_val = -1; 576 } 577 } 578 } 579 // No codec is registered, invalidate last audio decoder. 580 last_audio_decoder_ = -1; 581 return ret_val; 582 } 583 584 int AcmReceiver::RemoveCodec(uint8_t payload_type) { 585 int codec_index = PayloadType2CodecIndex(payload_type); 586 if (codec_index < 0) { // Such a payload-type is not registered. 587 return 0; 588 } 589 if (neteq_->RemovePayloadType(payload_type) != NetEq::kOK) { 590 LOG_FERR1(LS_ERROR, "AcmReceiver::RemoveCodec", payload_type); 591 return -1; 592 } 593 CriticalSectionScoped lock(crit_sect_.get()); 594 decoders_[codec_index].registered = false; 595 if (last_audio_decoder_ == codec_index) 596 last_audio_decoder_ = -1; // Codec is removed, invalidate last decoder. 597 return 0; 598 } 599 600 void AcmReceiver::set_id(int id) { 601 CriticalSectionScoped lock(crit_sect_.get()); 602 id_ = id; 603 } 604 605 bool AcmReceiver::GetPlayoutTimestamp(uint32_t* timestamp) { 606 if (av_sync_) { 607 assert(initial_delay_manager_.get()); 608 if (initial_delay_manager_->buffering()) { 609 return initial_delay_manager_->GetPlayoutTimestamp(timestamp); 610 } 611 } 612 return neteq_->GetPlayoutTimestamp(timestamp); 613 } 614 615 int AcmReceiver::last_audio_codec_id() const { 616 CriticalSectionScoped lock(crit_sect_.get()); 617 return last_audio_decoder_; 618 } 619 620 int AcmReceiver::last_audio_payload_type() const { 621 CriticalSectionScoped lock(crit_sect_.get()); 622 if (last_audio_decoder_ < 0) 623 return -1; 624 assert(decoders_[last_audio_decoder_].registered); 625 return decoders_[last_audio_decoder_].payload_type; 626 } 627 628 int AcmReceiver::RedPayloadType() const { 629 CriticalSectionScoped lock(crit_sect_.get()); 630 if (ACMCodecDB::kRED < 0 || 631 !decoders_[ACMCodecDB::kRED].registered) { 632 LOG_F(LS_WARNING) << "RED is not registered."; 633 return -1; 634 } 635 return decoders_[ACMCodecDB::kRED].payload_type; 636 } 637 638 int AcmReceiver::LastAudioCodec(CodecInst* codec) const { 639 CriticalSectionScoped lock(crit_sect_.get()); 640 if (last_audio_decoder_ < 0) { 641 return -1; 642 } 643 assert(decoders_[last_audio_decoder_].registered); 644 memcpy(codec, &ACMCodecDB::database_[last_audio_decoder_], sizeof(CodecInst)); 645 codec->pltype = decoders_[last_audio_decoder_].payload_type; 646 codec->channels = decoders_[last_audio_decoder_].channels; 647 return 0; 648 } 649 650 void AcmReceiver::NetworkStatistics(ACMNetworkStatistics* acm_stat) { 651 NetEqNetworkStatistics neteq_stat; 652 // NetEq function always returns zero, so we don't check the return value. 653 neteq_->NetworkStatistics(&neteq_stat); 654 655 acm_stat->currentBufferSize = neteq_stat.current_buffer_size_ms; 656 acm_stat->preferredBufferSize = neteq_stat.preferred_buffer_size_ms; 657 acm_stat->jitterPeaksFound = neteq_stat.jitter_peaks_found ? true : false; 658 acm_stat->currentPacketLossRate = neteq_stat.packet_loss_rate; 659 acm_stat->currentDiscardRate = neteq_stat.packet_discard_rate; 660 acm_stat->currentExpandRate = neteq_stat.expand_rate; 661 acm_stat->currentPreemptiveRate = neteq_stat.preemptive_rate; 662 acm_stat->currentAccelerateRate = neteq_stat.accelerate_rate; 663 acm_stat->clockDriftPPM = neteq_stat.clockdrift_ppm; 664 acm_stat->addedSamples = neteq_stat.added_zero_samples; 665 666 std::vector<int> waiting_times; 667 neteq_->WaitingTimes(&waiting_times); 668 size_t size = waiting_times.size(); 669 if (size == 0) { 670 acm_stat->meanWaitingTimeMs = -1; 671 acm_stat->medianWaitingTimeMs = -1; 672 acm_stat->minWaitingTimeMs = -1; 673 acm_stat->maxWaitingTimeMs = -1; 674 } else { 675 std::sort(waiting_times.begin(), waiting_times.end()); 676 if ((size & 0x1) == 0) { 677 acm_stat->medianWaitingTimeMs = (waiting_times[size / 2 - 1] + 678 waiting_times[size / 2]) / 2; 679 } else { 680 acm_stat->medianWaitingTimeMs = waiting_times[size / 2]; 681 } 682 acm_stat->minWaitingTimeMs = waiting_times.front(); 683 acm_stat->maxWaitingTimeMs = waiting_times.back(); 684 double sum = 0; 685 for (size_t i = 0; i < size; ++i) { 686 sum += waiting_times[i]; 687 } 688 acm_stat->meanWaitingTimeMs = static_cast<int>(sum / size); 689 } 690 } 691 692 int AcmReceiver::DecoderByPayloadType(uint8_t payload_type, 693 CodecInst* codec) const { 694 CriticalSectionScoped lock(crit_sect_.get()); 695 int codec_index = PayloadType2CodecIndex(payload_type); 696 if (codec_index < 0) { 697 LOG_FERR1(LS_ERROR, "AcmReceiver::DecoderByPayloadType", payload_type); 698 return -1; 699 } 700 memcpy(codec, &ACMCodecDB::database_[codec_index], sizeof(CodecInst)); 701 codec->pltype = decoders_[codec_index].payload_type; 702 codec->channels = decoders_[codec_index].channels; 703 return 0; 704 } 705 706 int AcmReceiver::PayloadType2CodecIndex(uint8_t payload_type) const { 707 for (int n = 0; n < ACMCodecDB::kMaxNumCodecs; ++n) { 708 if (decoders_[n].registered && decoders_[n].payload_type == payload_type) { 709 return n; 710 } 711 } 712 return -1; 713 } 714 715 int AcmReceiver::EnableNack(size_t max_nack_list_size) { 716 // Don't do anything if |max_nack_list_size| is out of range. 717 if (max_nack_list_size == 0 || max_nack_list_size > Nack::kNackListSizeLimit) 718 return -1; 719 720 CriticalSectionScoped lock(crit_sect_.get()); 721 if (!nack_enabled_) { 722 nack_.reset(Nack::Create(kNackThresholdPackets)); 723 nack_enabled_ = true; 724 725 // Sampling rate might need to be updated if we change from disable to 726 // enable. Do it if the receive codec is valid. 727 if (last_audio_decoder_ >= 0) { 728 nack_->UpdateSampleRate( 729 ACMCodecDB::database_[last_audio_decoder_].plfreq); 730 } 731 } 732 return nack_->SetMaxNackListSize(max_nack_list_size); 733 } 734 735 void AcmReceiver::DisableNack() { 736 CriticalSectionScoped lock(crit_sect_.get()); 737 nack_.reset(); // Memory is released. 738 nack_enabled_ = false; 739 } 740 741 std::vector<uint16_t> AcmReceiver::GetNackList( 742 int round_trip_time_ms) const { 743 CriticalSectionScoped lock(crit_sect_.get()); 744 if (round_trip_time_ms < 0) { 745 WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceAudioCoding, id_, 746 "GetNackList: round trip time cannot be negative." 747 " round_trip_time_ms=%d", round_trip_time_ms); 748 } 749 if (nack_enabled_ && round_trip_time_ms >= 0) { 750 assert(nack_.get()); 751 return nack_->GetNackList(round_trip_time_ms); 752 } 753 std::vector<uint16_t> empty_list; 754 return empty_list; 755 } 756 757 void AcmReceiver::ResetInitialDelay() { 758 { 759 CriticalSectionScoped lock(crit_sect_.get()); 760 av_sync_ = false; 761 initial_delay_manager_.reset(NULL); 762 missing_packets_sync_stream_.reset(NULL); 763 late_packets_sync_stream_.reset(NULL); 764 } 765 neteq_->SetMinimumDelay(0); 766 // TODO(turajs): Should NetEq Buffer be flushed? 767 } 768 769 // This function is called within critical section, no need to acquire a lock. 770 bool AcmReceiver::GetSilence(int desired_sample_rate_hz, AudioFrame* frame) { 771 assert(av_sync_); 772 assert(initial_delay_manager_.get()); 773 if (!initial_delay_manager_->buffering()) { 774 return false; 775 } 776 777 // We stop accumulating packets, if the number of packets or the total size 778 // exceeds a threshold. 779 int num_packets; 780 int max_num_packets; 781 const float kBufferingThresholdScale = 0.9f; 782 neteq_->PacketBufferStatistics(&num_packets, &max_num_packets); 783 if (num_packets > max_num_packets * kBufferingThresholdScale) { 784 initial_delay_manager_->DisableBuffering(); 785 return false; 786 } 787 788 // Update statistics. 789 call_stats_.DecodedBySilenceGenerator(); 790 791 // Set the values if already got a packet, otherwise set to default values. 792 if (last_audio_decoder_ >= 0) { 793 current_sample_rate_hz_ = ACMCodecDB::database_[last_audio_decoder_].plfreq; 794 frame->num_channels_ = decoders_[last_audio_decoder_].channels; 795 } else { 796 frame->num_channels_ = 1; 797 } 798 799 // Set the audio frame's sampling frequency. 800 if (desired_sample_rate_hz > 0) { 801 frame->sample_rate_hz_ = desired_sample_rate_hz; 802 } else { 803 frame->sample_rate_hz_ = current_sample_rate_hz_; 804 } 805 806 frame->samples_per_channel_ = frame->sample_rate_hz_ / 100; // Always 10 ms. 807 frame->speech_type_ = AudioFrame::kCNG; 808 frame->vad_activity_ = AudioFrame::kVadPassive; 809 int samples = frame->samples_per_channel_ * frame->num_channels_; 810 memset(frame->data_, 0, samples * sizeof(int16_t)); 811 return true; 812 } 813 814 NetEqBackgroundNoiseMode AcmReceiver::BackgroundNoiseModeForTest() const { 815 return neteq_->BackgroundNoiseMode(); 816 } 817 818 int AcmReceiver::RtpHeaderToCodecIndex( 819 const RTPHeader &rtp_header, const uint8_t* payload) const { 820 uint8_t payload_type = rtp_header.payloadType; 821 if (ACMCodecDB::kRED >= 0 && // This ensures that RED is defined in WebRTC. 822 decoders_[ACMCodecDB::kRED].registered && 823 payload_type == decoders_[ACMCodecDB::kRED].payload_type) { 824 // This is a RED packet, get the payload of the audio codec. 825 payload_type = payload[0] & 0x7F; 826 } 827 828 // Check if the payload is registered. 829 return PayloadType2CodecIndex(payload_type); 830 } 831 832 uint32_t AcmReceiver::NowInTimestamp(int decoder_sampling_rate) const { 833 // Down-cast the time to (32-6)-bit since we only care about 834 // the least significant bits. (32-6) bits cover 2^(32-6) = 67108864 ms. 835 // We masked 6 most significant bits of 32-bit so there is no overflow in 836 // the conversion from milliseconds to timestamp. 837 const uint32_t now_in_ms = static_cast<uint32_t>( 838 clock_->TimeInMilliseconds() & 0x03ffffff); 839 return static_cast<uint32_t>( 840 (decoder_sampling_rate / 1000) * now_in_ms); 841 } 842 843 // This function only interacts with |neteq_|, therefore, it does not have to 844 // be within critical section of AcmReceiver. It is inserting packets 845 // into NetEq, so we call it when |decode_lock_| is acquired. However, this is 846 // not essential as sync-packets do not interact with codecs (especially BWE). 847 void AcmReceiver::InsertStreamOfSyncPackets( 848 InitialDelayManager::SyncStream* sync_stream) { 849 assert(sync_stream); 850 assert(av_sync_); 851 for (int n = 0; n < sync_stream->num_sync_packets; ++n) { 852 neteq_->InsertSyncPacket(sync_stream->rtp_info, 853 sync_stream->receive_timestamp); 854 ++sync_stream->rtp_info.header.sequenceNumber; 855 sync_stream->rtp_info.header.timestamp += sync_stream->timestamp_step; 856 sync_stream->receive_timestamp += sync_stream->timestamp_step; 857 } 858 } 859 860 void AcmReceiver::GetDecodingCallStatistics( 861 AudioDecodingCallStats* stats) const { 862 CriticalSectionScoped lock(crit_sect_.get()); 863 *stats = call_stats_.GetDecodingStatistics(); 864 } 865 866 } // namespace acm2 867 868 } // namespace webrtc 869