1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef WEBRTC_MODULES_AUDIO_CODING_MAIN_ACM2_AUDIO_CODING_MODULE_IMPL_H_ 12 #define WEBRTC_MODULES_AUDIO_CODING_MAIN_ACM2_AUDIO_CODING_MODULE_IMPL_H_ 13 14 #include <vector> 15 16 #include "webrtc/base/thread_annotations.h" 17 #include "webrtc/common_types.h" 18 #include "webrtc/engine_configurations.h" 19 #include "webrtc/modules/audio_coding/main/acm2/acm_codec_database.h" 20 #include "webrtc/modules/audio_coding/main/acm2/acm_receiver.h" 21 #include "webrtc/modules/audio_coding/main/acm2/acm_resampler.h" 22 #include "webrtc/system_wrappers/interface/scoped_ptr.h" 23 24 namespace webrtc { 25 26 class CriticalSectionWrapper; 27 28 namespace acm2 { 29 30 class ACMDTMFDetection; 31 class ACMGenericCodec; 32 33 class AudioCodingModuleImpl : public AudioCodingModule { 34 public: 35 explicit AudioCodingModuleImpl(const AudioCodingModule::Config& config); 36 ~AudioCodingModuleImpl(); 37 38 // Change the unique identifier of this object. 39 virtual int32_t ChangeUniqueId(const int32_t id) OVERRIDE; 40 41 // Returns the number of milliseconds until the module want a worker thread 42 // to call Process. 43 virtual int32_t TimeUntilNextProcess() OVERRIDE; 44 45 // Process any pending tasks such as timeouts. 46 virtual int32_t Process() OVERRIDE; 47 48 ///////////////////////////////////////// 49 // Sender 50 // 51 52 // Initialize send codec. 53 virtual int InitializeSender() OVERRIDE; 54 55 // Reset send codec. 56 virtual int ResetEncoder() OVERRIDE; 57 58 // Can be called multiple times for Codec, CNG, RED. 59 virtual int RegisterSendCodec(const CodecInst& send_codec) OVERRIDE; 60 61 // Register Secondary codec for dual-streaming. Dual-streaming is activated 62 // right after the secondary codec is registered. 63 virtual int RegisterSecondarySendCodec(const CodecInst& send_codec) OVERRIDE; 64 65 // Unregister the secondary codec. Dual-streaming is deactivated right after 66 // deregistering secondary codec. 67 virtual void UnregisterSecondarySendCodec() OVERRIDE; 68 69 // Get the secondary codec. 70 virtual int SecondarySendCodec(CodecInst* secondary_codec) const OVERRIDE; 71 72 // Get current send codec. 73 virtual int SendCodec(CodecInst* current_codec) const OVERRIDE; 74 75 // Get current send frequency. 76 virtual int SendFrequency() const OVERRIDE; 77 78 // Get encode bit-rate. 79 // Adaptive rate codecs return their current encode target rate, while other 80 // codecs return there long-term average or their fixed rate. 81 virtual int SendBitrate() const OVERRIDE; 82 83 // Set available bandwidth, inform the encoder about the 84 // estimated bandwidth received from the remote party. 85 virtual int SetReceivedEstimatedBandwidth(int bw) OVERRIDE; 86 87 // Register a transport callback which will be 88 // called to deliver the encoded buffers. 89 virtual int RegisterTransportCallback( 90 AudioPacketizationCallback* transport) OVERRIDE; 91 92 // Add 10 ms of raw (PCM) audio data to the encoder. 93 virtual int Add10MsData(const AudioFrame& audio_frame) OVERRIDE; 94 95 ///////////////////////////////////////// 96 // (RED) Redundant Coding 97 // 98 99 // Configure RED status i.e. on/off. 100 virtual int SetREDStatus(bool enable_red) OVERRIDE; 101 102 // Get RED status. 103 virtual bool REDStatus() const OVERRIDE; 104 105 ///////////////////////////////////////// 106 // (FEC) Forward Error Correction (codec internal) 107 // 108 109 // Configure FEC status i.e. on/off. 110 virtual int SetCodecFEC(bool enabled_codec_fec) OVERRIDE; 111 112 // Get FEC status. 113 virtual bool CodecFEC() const OVERRIDE; 114 115 // Set target packet loss rate 116 virtual int SetPacketLossRate(int loss_rate) OVERRIDE; 117 118 ///////////////////////////////////////// 119 // (VAD) Voice Activity Detection 120 // and 121 // (CNG) Comfort Noise Generation 122 // 123 124 virtual int SetVAD(bool enable_dtx = true, 125 bool enable_vad = false, 126 ACMVADMode mode = VADNormal) OVERRIDE; 127 128 virtual int VAD(bool* dtx_enabled, 129 bool* vad_enabled, 130 ACMVADMode* mode) const OVERRIDE; 131 132 virtual int RegisterVADCallback(ACMVADCallback* vad_callback) OVERRIDE; 133 134 ///////////////////////////////////////// 135 // Receiver 136 // 137 138 // Initialize receiver, resets codec database etc. 139 virtual int InitializeReceiver() OVERRIDE; 140 141 // Reset the decoder state. 142 virtual int ResetDecoder() OVERRIDE; 143 144 // Get current receive frequency. 145 virtual int ReceiveFrequency() const OVERRIDE; 146 147 // Get current playout frequency. 148 virtual int PlayoutFrequency() const OVERRIDE; 149 150 // Register possible receive codecs, can be called multiple times, 151 // for codecs, CNG, DTMF, RED. 152 virtual int RegisterReceiveCodec(const CodecInst& receive_codec) OVERRIDE; 153 154 // Get current received codec. 155 virtual int ReceiveCodec(CodecInst* current_codec) const OVERRIDE; 156 157 // Incoming packet from network parsed and ready for decode. 158 virtual int IncomingPacket(const uint8_t* incoming_payload, 159 int payload_length, 160 const WebRtcRTPHeader& rtp_info) OVERRIDE; 161 162 // Incoming payloads, without rtp-info, the rtp-info will be created in ACM. 163 // One usage for this API is when pre-encoded files are pushed in ACM. 164 virtual int IncomingPayload(const uint8_t* incoming_payload, 165 int payload_length, 166 uint8_t payload_type, 167 uint32_t timestamp) OVERRIDE; 168 169 // Minimum playout delay. 170 virtual int SetMinimumPlayoutDelay(int time_ms) OVERRIDE; 171 172 // Maximum playout delay. 173 virtual int SetMaximumPlayoutDelay(int time_ms) OVERRIDE; 174 175 // Smallest latency NetEq will maintain. 176 virtual int LeastRequiredDelayMs() const OVERRIDE; 177 178 // Impose an initial delay on playout. ACM plays silence until |delay_ms| 179 // audio is accumulated in NetEq buffer, then starts decoding payloads. 180 virtual int SetInitialPlayoutDelay(int delay_ms) OVERRIDE; 181 182 // TODO(turajs): DTMF playout is always activated in NetEq these APIs should 183 // be removed, as well as all VoE related APIs and methods. 184 // 185 // Configure Dtmf playout status i.e on/off playout the incoming outband Dtmf 186 // tone. 187 virtual int SetDtmfPlayoutStatus(bool enable) OVERRIDE { return 0; } 188 189 // Get Dtmf playout status. 190 virtual bool DtmfPlayoutStatus() const OVERRIDE { return true; } 191 192 // Estimate the Bandwidth based on the incoming stream, needed 193 // for one way audio where the RTCP send the BW estimate. 194 // This is also done in the RTP module . 195 virtual int DecoderEstimatedBandwidth() const OVERRIDE; 196 197 // Set playout mode voice, fax. 198 virtual int SetPlayoutMode(AudioPlayoutMode mode) OVERRIDE; 199 200 // Get playout mode voice, fax. 201 virtual AudioPlayoutMode PlayoutMode() const OVERRIDE; 202 203 // Get playout timestamp. 204 virtual int PlayoutTimestamp(uint32_t* timestamp) OVERRIDE; 205 206 // Get 10 milliseconds of raw audio data to play out, and 207 // automatic resample to the requested frequency if > 0. 208 virtual int PlayoutData10Ms(int desired_freq_hz, 209 AudioFrame* audio_frame) OVERRIDE; 210 211 ///////////////////////////////////////// 212 // Statistics 213 // 214 215 virtual int NetworkStatistics(ACMNetworkStatistics* statistics) OVERRIDE; 216 217 // GET RED payload for iSAC. The method id called when 'this' ACM is 218 // the default ACM. 219 // TODO(henrik.lundin) Not used. Remove? 220 int REDPayloadISAC(int isac_rate, 221 int isac_bw_estimate, 222 uint8_t* payload, 223 int16_t* length_bytes); 224 225 virtual int ReplaceInternalDTXWithWebRtc(bool use_webrtc_dtx) OVERRIDE; 226 227 virtual int IsInternalDTXReplacedWithWebRtc(bool* uses_webrtc_dtx) OVERRIDE; 228 229 virtual int SetISACMaxRate(int max_bit_per_sec) OVERRIDE; 230 231 virtual int SetISACMaxPayloadSize(int max_size_bytes) OVERRIDE; 232 233 virtual int ConfigISACBandwidthEstimator( 234 int frame_size_ms, 235 int rate_bit_per_sec, 236 bool enforce_frame_size = false) OVERRIDE; 237 238 // If current send codec is Opus, informs it about the maximum playback rate 239 // the receiver will render. 240 virtual int SetOpusMaxPlaybackRate(int frequency_hz) OVERRIDE; 241 242 virtual int UnregisterReceiveCodec(uint8_t payload_type) OVERRIDE; 243 244 virtual int EnableNack(size_t max_nack_list_size) OVERRIDE; 245 246 virtual void DisableNack() OVERRIDE; 247 248 virtual std::vector<uint16_t> GetNackList( 249 int round_trip_time_ms) const OVERRIDE; 250 251 virtual void GetDecodingCallStatistics( 252 AudioDecodingCallStats* stats) const OVERRIDE; 253 254 private: 255 int UnregisterReceiveCodecSafe(int payload_type); 256 257 ACMGenericCodec* CreateCodec(const CodecInst& codec); 258 259 int InitializeReceiverSafe() EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); 260 261 bool HaveValidEncoder(const char* caller_name) const 262 EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); 263 264 // Set VAD/DTX status. This function does not acquire a lock, and it is 265 // created to be called only from inside a critical section. 266 int SetVADSafe(bool enable_dtx, bool enable_vad, ACMVADMode mode) 267 EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); 268 269 // Process buffered audio when dual-streaming is not enabled (When RED is 270 // enabled still this function is used.) 271 int ProcessSingleStream(); 272 273 // Process buffered audio when dual-streaming is enabled, i.e. secondary send 274 // codec is registered. 275 int ProcessDualStream(); 276 277 // Preprocessing of input audio, including resampling and down-mixing if 278 // required, before pushing audio into encoder's buffer. 279 // 280 // in_frame: input audio-frame 281 // ptr_out: pointer to output audio_frame. If no preprocessing is required 282 // |ptr_out| will be pointing to |in_frame|, otherwise pointing to 283 // |preprocess_frame_|. 284 // 285 // Return value: 286 // -1: if encountering an error. 287 // 0: otherwise. 288 int PreprocessToAddData(const AudioFrame& in_frame, 289 const AudioFrame** ptr_out) 290 EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); 291 292 // Change required states after starting to receive the codec corresponding 293 // to |index|. 294 int UpdateUponReceivingCodec(int index); 295 296 int EncodeFragmentation(int fragmentation_index, 297 int payload_type, 298 uint32_t current_timestamp, 299 ACMGenericCodec* encoder, 300 uint8_t* stream) 301 EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); 302 303 void ResetFragmentation(int vector_size) 304 EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); 305 306 // Get a pointer to AudioDecoder of the given codec. For some codecs, e.g. 307 // iSAC, encoding and decoding have to be performed on a shared 308 // codec-instance. By calling this method, we get the codec-instance that ACM 309 // owns, then pass that to NetEq. This way, we perform both encoding and 310 // decoding on the same codec-instance. Furthermore, ACM would have control 311 // over decoder functionality if required. If |codec| does not share an 312 // instance between encoder and decoder, the |*decoder| is set NULL. 313 // The field ACMCodecDB::CodecSettings.owns_decoder indicates that if a 314 // codec owns the decoder-instance. For such codecs |*decoder| should be a 315 // valid pointer, otherwise it will be NULL. 316 int GetAudioDecoder(const CodecInst& codec, int codec_id, 317 int mirror_id, AudioDecoder** decoder) 318 EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); 319 320 CriticalSectionWrapper* acm_crit_sect_; 321 int id_; // TODO(henrik.lundin) Make const. 322 uint32_t expected_codec_ts_ GUARDED_BY(acm_crit_sect_); 323 uint32_t expected_in_ts_ GUARDED_BY(acm_crit_sect_); 324 CodecInst send_codec_inst_ GUARDED_BY(acm_crit_sect_); 325 326 uint8_t cng_nb_pltype_ GUARDED_BY(acm_crit_sect_); 327 uint8_t cng_wb_pltype_ GUARDED_BY(acm_crit_sect_); 328 uint8_t cng_swb_pltype_ GUARDED_BY(acm_crit_sect_); 329 uint8_t cng_fb_pltype_ GUARDED_BY(acm_crit_sect_); 330 331 uint8_t red_pltype_ GUARDED_BY(acm_crit_sect_); 332 bool vad_enabled_ GUARDED_BY(acm_crit_sect_); 333 bool dtx_enabled_ GUARDED_BY(acm_crit_sect_); 334 ACMVADMode vad_mode_ GUARDED_BY(acm_crit_sect_); 335 ACMGenericCodec* codecs_[ACMCodecDB::kMaxNumCodecs] 336 GUARDED_BY(acm_crit_sect_); 337 int mirror_codec_idx_[ACMCodecDB::kMaxNumCodecs] GUARDED_BY(acm_crit_sect_); 338 bool stereo_send_ GUARDED_BY(acm_crit_sect_); 339 int current_send_codec_idx_ GUARDED_BY(acm_crit_sect_); 340 bool send_codec_registered_ GUARDED_BY(acm_crit_sect_); 341 ACMResampler resampler_ GUARDED_BY(acm_crit_sect_); 342 AcmReceiver receiver_; // AcmReceiver has it's own internal lock. 343 344 // RED. 345 bool is_first_red_ GUARDED_BY(acm_crit_sect_); 346 bool red_enabled_ GUARDED_BY(acm_crit_sect_); 347 348 // TODO(turajs): |red_buffer_| is allocated in constructor, why having them 349 // as pointers and not an array. If concerned about the memory, then make a 350 // set-up function to allocate them only when they are going to be used, i.e. 351 // RED or Dual-streaming is enabled. 352 uint8_t* red_buffer_ GUARDED_BY(acm_crit_sect_); 353 354 // TODO(turajs): we actually don't need |fragmentation_| as a member variable. 355 // It is sufficient to keep the length & payload type of previous payload in 356 // member variables. 357 RTPFragmentationHeader fragmentation_ GUARDED_BY(acm_crit_sect_); 358 uint32_t last_red_timestamp_ GUARDED_BY(acm_crit_sect_); 359 360 // Codec internal FEC 361 bool codec_fec_enabled_ GUARDED_BY(acm_crit_sect_); 362 363 // This is to keep track of CN instances where we can send DTMFs. 364 uint8_t previous_pltype_ GUARDED_BY(acm_crit_sect_); 365 366 // Used when payloads are pushed into ACM without any RTP info 367 // One example is when pre-encoded bit-stream is pushed from 368 // a file. 369 // IMPORTANT: this variable is only used in IncomingPayload(), therefore, 370 // no lock acquired when interacting with this variable. If it is going to 371 // be used in other methods, locks need to be taken. 372 WebRtcRTPHeader* aux_rtp_header_; 373 374 bool receiver_initialized_ GUARDED_BY(acm_crit_sect_); 375 376 AudioFrame preprocess_frame_ GUARDED_BY(acm_crit_sect_); 377 CodecInst secondary_send_codec_inst_ GUARDED_BY(acm_crit_sect_); 378 scoped_ptr<ACMGenericCodec> secondary_encoder_ GUARDED_BY(acm_crit_sect_); 379 uint32_t codec_timestamp_ GUARDED_BY(acm_crit_sect_); 380 bool first_10ms_data_ GUARDED_BY(acm_crit_sect_); 381 382 CriticalSectionWrapper* callback_crit_sect_; 383 AudioPacketizationCallback* packetization_callback_ 384 GUARDED_BY(callback_crit_sect_); 385 ACMVADCallback* vad_callback_ GUARDED_BY(callback_crit_sect_); 386 }; 387 388 } // namespace acm2 389 390 class AudioCodingImpl : public AudioCoding { 391 public: 392 AudioCodingImpl(const Config& config) { 393 AudioCodingModule::Config config_old = config.ToOldConfig(); 394 acm_old_.reset(new acm2::AudioCodingModuleImpl(config_old)); 395 acm_old_->RegisterTransportCallback(config.transport); 396 acm_old_->RegisterVADCallback(config.vad_callback); 397 acm_old_->SetDtmfPlayoutStatus(config.play_dtmf); 398 if (config.initial_playout_delay_ms > 0) { 399 acm_old_->SetInitialPlayoutDelay(config.initial_playout_delay_ms); 400 } 401 playout_frequency_hz_ = config.playout_frequency_hz; 402 } 403 404 virtual ~AudioCodingImpl() OVERRIDE {}; 405 406 virtual bool RegisterSendCodec(AudioEncoder* send_codec) OVERRIDE; 407 408 virtual bool RegisterSendCodec(int encoder_type, 409 uint8_t payload_type, 410 int frame_size_samples = 0) OVERRIDE; 411 412 virtual const AudioEncoder* GetSenderInfo() const OVERRIDE; 413 414 virtual const CodecInst* GetSenderCodecInst() OVERRIDE; 415 416 virtual int Add10MsAudio(const AudioFrame& audio_frame) OVERRIDE; 417 418 virtual const ReceiverInfo* GetReceiverInfo() const OVERRIDE; 419 420 virtual bool RegisterReceiveCodec(AudioDecoder* receive_codec) OVERRIDE; 421 422 virtual bool RegisterReceiveCodec(int decoder_type, 423 uint8_t payload_type) OVERRIDE; 424 425 virtual bool InsertPacket(const uint8_t* incoming_payload, 426 int32_t payload_len_bytes, 427 const WebRtcRTPHeader& rtp_info) OVERRIDE; 428 429 virtual bool InsertPayload(const uint8_t* incoming_payload, 430 int32_t payload_len_byte, 431 uint8_t payload_type, 432 uint32_t timestamp) OVERRIDE; 433 434 virtual bool SetMinimumPlayoutDelay(int time_ms) OVERRIDE; 435 436 virtual bool SetMaximumPlayoutDelay(int time_ms) OVERRIDE; 437 438 virtual int LeastRequiredDelayMs() const OVERRIDE; 439 440 virtual bool PlayoutTimestamp(uint32_t* timestamp) OVERRIDE; 441 442 virtual bool Get10MsAudio(AudioFrame* audio_frame) OVERRIDE; 443 444 virtual bool NetworkStatistics( 445 ACMNetworkStatistics* network_statistics) OVERRIDE; 446 447 virtual bool EnableNack(size_t max_nack_list_size) OVERRIDE; 448 449 virtual void DisableNack() OVERRIDE; 450 451 virtual bool SetVad(bool enable_dtx, 452 bool enable_vad, 453 ACMVADMode vad_mode) OVERRIDE; 454 455 virtual std::vector<uint16_t> GetNackList( 456 int round_trip_time_ms) const OVERRIDE; 457 458 virtual void GetDecodingCallStatistics( 459 AudioDecodingCallStats* call_stats) const OVERRIDE; 460 461 private: 462 // Temporary method to be used during redesign phase. 463 // Maps |codec_type| (a value from the anonymous enum in acm2::ACMCodecDB) to 464 // |codec_name|, |sample_rate_hz|, and |channels|. 465 // TODO(henrik.lundin) Remove this when no longer needed. 466 static bool MapCodecTypeToParameters(int codec_type, 467 std::string* codec_name, 468 int* sample_rate_hz, 469 int* channels); 470 471 int playout_frequency_hz_; 472 // TODO(henrik.lundin): All members below this line are temporary and should 473 // be removed after refactoring is completed. 474 scoped_ptr<acm2::AudioCodingModuleImpl> acm_old_; 475 CodecInst current_send_codec_; 476 }; 477 478 } // namespace webrtc 479 480 #endif // WEBRTC_MODULES_AUDIO_CODING_MAIN_ACM2_AUDIO_CODING_MODULE_IMPL_H_ 481
