1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ 12 #define WEBRTC_MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ 13 14 #include <vector> 15 16 #include "webrtc/base/constructormagic.h" 17 #include "webrtc/modules/audio_coding/neteq/audio_multi_vector.h" 18 #include "webrtc/modules/audio_coding/neteq/defines.h" 19 #include "webrtc/modules/audio_coding/neteq/interface/neteq.h" 20 #include "webrtc/modules/audio_coding/neteq/packet.h" // Declare PacketList. 21 #include "webrtc/modules/audio_coding/neteq/random_vector.h" 22 #include "webrtc/modules/audio_coding/neteq/rtcp.h" 23 #include "webrtc/modules/audio_coding/neteq/statistics_calculator.h" 24 #include "webrtc/system_wrappers/interface/scoped_ptr.h" 25 #include "webrtc/system_wrappers/interface/thread_annotations.h" 26 #include "webrtc/typedefs.h" 27 28 namespace webrtc { 29 30 // Forward declarations. 31 class Accelerate; 32 class BackgroundNoise; 33 class BufferLevelFilter; 34 class ComfortNoise; 35 class CriticalSectionWrapper; 36 class DecisionLogic; 37 class DecoderDatabase; 38 class DelayManager; 39 class DelayPeakDetector; 40 class DtmfBuffer; 41 class DtmfToneGenerator; 42 class Expand; 43 class Merge; 44 class Normal; 45 class PacketBuffer; 46 class PayloadSplitter; 47 class PostDecodeVad; 48 class PreemptiveExpand; 49 class RandomVector; 50 class SyncBuffer; 51 class TimestampScaler; 52 struct AccelerateFactory; 53 struct DtmfEvent; 54 struct ExpandFactory; 55 struct PreemptiveExpandFactory; 56 57 class NetEqImpl : public webrtc::NetEq { 58 public: 59 // Creates a new NetEqImpl object. The object will assume ownership of all 60 // injected dependencies, and will delete them when done. 61 NetEqImpl(int fs, 62 BufferLevelFilter* buffer_level_filter, 63 DecoderDatabase* decoder_database, 64 DelayManager* delay_manager, 65 DelayPeakDetector* delay_peak_detector, 66 DtmfBuffer* dtmf_buffer, 67 DtmfToneGenerator* dtmf_tone_generator, 68 PacketBuffer* packet_buffer, 69 PayloadSplitter* payload_splitter, 70 TimestampScaler* timestamp_scaler, 71 AccelerateFactory* accelerate_factory, 72 ExpandFactory* expand_factory, 73 PreemptiveExpandFactory* preemptive_expand_factory, 74 bool create_components = true); 75 76 virtual ~NetEqImpl(); 77 78 // Inserts a new packet into NetEq. The |receive_timestamp| is an indication 79 // of the time when the packet was received, and should be measured with 80 // the same tick rate as the RTP timestamp of the current payload. 81 // Returns 0 on success, -1 on failure. 82 virtual int InsertPacket(const WebRtcRTPHeader& rtp_header, 83 const uint8_t* payload, 84 int length_bytes, 85 uint32_t receive_timestamp); 86 87 // Inserts a sync-packet into packet queue. Sync-packets are decoded to 88 // silence and are intended to keep AV-sync intact in an event of long packet 89 // losses when Video NACK is enabled but Audio NACK is not. Clients of NetEq 90 // might insert sync-packet when they observe that buffer level of NetEq is 91 // decreasing below a certain threshold, defined by the application. 92 // Sync-packets should have the same payload type as the last audio payload 93 // type, i.e. they cannot have DTMF or CNG payload type, nor a codec change 94 // can be implied by inserting a sync-packet. 95 // Returns kOk on success, kFail on failure. 96 virtual int InsertSyncPacket(const WebRtcRTPHeader& rtp_header, 97 uint32_t receive_timestamp); 98 99 // Instructs NetEq to deliver 10 ms of audio data. The data is written to 100 // |output_audio|, which can hold (at least) |max_length| elements. 101 // The number of channels that were written to the output is provided in 102 // the output variable |num_channels|, and each channel contains 103 // |samples_per_channel| elements. If more than one channel is written, 104 // the samples are interleaved. 105 // The speech type is written to |type|, if |type| is not NULL. 106 // Returns kOK on success, or kFail in case of an error. 107 virtual int GetAudio(size_t max_length, int16_t* output_audio, 108 int* samples_per_channel, int* num_channels, 109 NetEqOutputType* type); 110 111 // Associates |rtp_payload_type| with |codec| and stores the information in 112 // the codec database. Returns kOK on success, kFail on failure. 113 virtual int RegisterPayloadType(enum NetEqDecoder codec, 114 uint8_t rtp_payload_type); 115 116 // Provides an externally created decoder object |decoder| to insert in the 117 // decoder database. The decoder implements a decoder of type |codec| and 118 // associates it with |rtp_payload_type|. Returns kOK on success, kFail on 119 // failure. 120 virtual int RegisterExternalDecoder(AudioDecoder* decoder, 121 enum NetEqDecoder codec, 122 uint8_t rtp_payload_type); 123 124 // Removes |rtp_payload_type| from the codec database. Returns 0 on success, 125 // -1 on failure. 126 virtual int RemovePayloadType(uint8_t rtp_payload_type); 127 128 virtual bool SetMinimumDelay(int delay_ms); 129 130 virtual bool SetMaximumDelay(int delay_ms); 131 132 virtual int LeastRequiredDelayMs() const; 133 134 virtual int SetTargetDelay() { return kNotImplemented; } 135 136 virtual int TargetDelay() { return kNotImplemented; } 137 138 virtual int CurrentDelay() { return kNotImplemented; } 139 140 // Sets the playout mode to |mode|. 141 virtual void SetPlayoutMode(NetEqPlayoutMode mode); 142 143 // Returns the current playout mode. 144 virtual NetEqPlayoutMode PlayoutMode() const; 145 146 // Writes the current network statistics to |stats|. The statistics are reset 147 // after the call. 148 virtual int NetworkStatistics(NetEqNetworkStatistics* stats); 149 150 // Writes the last packet waiting times (in ms) to |waiting_times|. The number 151 // of values written is no more than 100, but may be smaller if the interface 152 // is polled again before 100 packets has arrived. 153 virtual void WaitingTimes(std::vector<int>* waiting_times); 154 155 // Writes the current RTCP statistics to |stats|. The statistics are reset 156 // and a new report period is started with the call. 157 virtual void GetRtcpStatistics(RtcpStatistics* stats); 158 159 // Same as RtcpStatistics(), but does not reset anything. 160 virtual void GetRtcpStatisticsNoReset(RtcpStatistics* stats); 161 162 // Enables post-decode VAD. When enabled, GetAudio() will return 163 // kOutputVADPassive when the signal contains no speech. 164 virtual void EnableVad(); 165 166 // Disables post-decode VAD. 167 virtual void DisableVad(); 168 169 virtual bool GetPlayoutTimestamp(uint32_t* timestamp); 170 171 virtual int SetTargetNumberOfChannels() { return kNotImplemented; } 172 173 virtual int SetTargetSampleRate() { return kNotImplemented; } 174 175 // Returns the error code for the last occurred error. If no error has 176 // occurred, 0 is returned. 177 virtual int LastError(); 178 179 // Returns the error code last returned by a decoder (audio or comfort noise). 180 // When LastError() returns kDecoderErrorCode or kComfortNoiseErrorCode, check 181 // this method to get the decoder's error code. 182 virtual int LastDecoderError(); 183 184 // Flushes both the packet buffer and the sync buffer. 185 virtual void FlushBuffers(); 186 187 virtual void PacketBufferStatistics(int* current_num_packets, 188 int* max_num_packets) const; 189 190 // Get sequence number and timestamp of the latest RTP. 191 // This method is to facilitate NACK. 192 virtual int DecodedRtpInfo(int* sequence_number, uint32_t* timestamp) const; 193 194 // Sets background noise mode. 195 virtual void SetBackgroundNoiseMode(NetEqBackgroundNoiseMode mode); 196 197 // Gets background noise mode. 198 virtual NetEqBackgroundNoiseMode BackgroundNoiseMode() const; 199 200 // This accessor method is only intended for testing purposes. 201 virtual const SyncBuffer* sync_buffer_for_test() const; 202 203 protected: 204 static const int kOutputSizeMs = 10; 205 static const int kMaxFrameSize = 2880; // 60 ms @ 48 kHz. 206 // TODO(hlundin): Provide a better value for kSyncBufferSize. 207 static const int kSyncBufferSize = 2 * kMaxFrameSize; 208 209 // Inserts a new packet into NetEq. This is used by the InsertPacket method 210 // above. Returns 0 on success, otherwise an error code. 211 // TODO(hlundin): Merge this with InsertPacket above? 212 int InsertPacketInternal(const WebRtcRTPHeader& rtp_header, 213 const uint8_t* payload, 214 int length_bytes, 215 uint32_t receive_timestamp, 216 bool is_sync_packet) 217 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 218 219 // Delivers 10 ms of audio data. The data is written to |output|, which can 220 // hold (at least) |max_length| elements. The number of channels that were 221 // written to the output is provided in the output variable |num_channels|, 222 // and each channel contains |samples_per_channel| elements. If more than one 223 // channel is written, the samples are interleaved. 224 // Returns 0 on success, otherwise an error code. 225 int GetAudioInternal(size_t max_length, 226 int16_t* output, 227 int* samples_per_channel, 228 int* num_channels) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 229 230 // Provides a decision to the GetAudioInternal method. The decision what to 231 // do is written to |operation|. Packets to decode are written to 232 // |packet_list|, and a DTMF event to play is written to |dtmf_event|. When 233 // DTMF should be played, |play_dtmf| is set to true by the method. 234 // Returns 0 on success, otherwise an error code. 235 int GetDecision(Operations* operation, 236 PacketList* packet_list, 237 DtmfEvent* dtmf_event, 238 bool* play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 239 240 // Decodes the speech packets in |packet_list|, and writes the results to 241 // |decoded_buffer|, which is allocated to hold |decoded_buffer_length| 242 // elements. The length of the decoded data is written to |decoded_length|. 243 // The speech type -- speech or (codec-internal) comfort noise -- is written 244 // to |speech_type|. If |packet_list| contains any SID frames for RFC 3389 245 // comfort noise, those are not decoded. 246 int Decode(PacketList* packet_list, 247 Operations* operation, 248 int* decoded_length, 249 AudioDecoder::SpeechType* speech_type) 250 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 251 252 // Sub-method to Decode(). Performs the actual decoding. 253 int DecodeLoop(PacketList* packet_list, 254 Operations* operation, 255 AudioDecoder* decoder, 256 int* decoded_length, 257 AudioDecoder::SpeechType* speech_type) 258 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 259 260 // Sub-method which calls the Normal class to perform the normal operation. 261 void DoNormal(const int16_t* decoded_buffer, 262 size_t decoded_length, 263 AudioDecoder::SpeechType speech_type, 264 bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 265 266 // Sub-method which calls the Merge class to perform the merge operation. 267 void DoMerge(int16_t* decoded_buffer, 268 size_t decoded_length, 269 AudioDecoder::SpeechType speech_type, 270 bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 271 272 // Sub-method which calls the Expand class to perform the expand operation. 273 int DoExpand(bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 274 275 // Sub-method which calls the Accelerate class to perform the accelerate 276 // operation. 277 int DoAccelerate(int16_t* decoded_buffer, 278 size_t decoded_length, 279 AudioDecoder::SpeechType speech_type, 280 bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 281 282 // Sub-method which calls the PreemptiveExpand class to perform the 283 // preemtive expand operation. 284 int DoPreemptiveExpand(int16_t* decoded_buffer, 285 size_t decoded_length, 286 AudioDecoder::SpeechType speech_type, 287 bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 288 289 // Sub-method which calls the ComfortNoise class to generate RFC 3389 comfort 290 // noise. |packet_list| can either contain one SID frame to update the 291 // noise parameters, or no payload at all, in which case the previously 292 // received parameters are used. 293 int DoRfc3389Cng(PacketList* packet_list, bool play_dtmf) 294 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 295 296 // Calls the audio decoder to generate codec-internal comfort noise when 297 // no packet was received. 298 void DoCodecInternalCng() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 299 300 // Calls the DtmfToneGenerator class to generate DTMF tones. 301 int DoDtmf(const DtmfEvent& dtmf_event, bool* play_dtmf) 302 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 303 304 // Produces packet-loss concealment using alternative methods. If the codec 305 // has an internal PLC, it is called to generate samples. Otherwise, the 306 // method performs zero-stuffing. 307 void DoAlternativePlc(bool increase_timestamp) 308 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 309 310 // Overdub DTMF on top of |output|. 311 int DtmfOverdub(const DtmfEvent& dtmf_event, 312 size_t num_channels, 313 int16_t* output) const EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 314 315 // Extracts packets from |packet_buffer_| to produce at least 316 // |required_samples| samples. The packets are inserted into |packet_list|. 317 // Returns the number of samples that the packets in the list will produce, or 318 // -1 in case of an error. 319 int ExtractPackets(int required_samples, PacketList* packet_list) 320 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 321 322 // Resets various variables and objects to new values based on the sample rate 323 // |fs_hz| and |channels| number audio channels. 324 void SetSampleRateAndChannels(int fs_hz, size_t channels) 325 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 326 327 // Returns the output type for the audio produced by the latest call to 328 // GetAudio(). 329 NetEqOutputType LastOutputType() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 330 331 // Updates Expand and Merge. 332 virtual void UpdatePlcComponents(int fs_hz, size_t channels) 333 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 334 335 // Creates DecisionLogic object for the given mode. 336 virtual void CreateDecisionLogic(NetEqPlayoutMode mode) 337 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 338 339 const scoped_ptr<CriticalSectionWrapper> crit_sect_; 340 const scoped_ptr<BufferLevelFilter> buffer_level_filter_ 341 GUARDED_BY(crit_sect_); 342 const scoped_ptr<DecoderDatabase> decoder_database_ GUARDED_BY(crit_sect_); 343 const scoped_ptr<DelayManager> delay_manager_ GUARDED_BY(crit_sect_); 344 const scoped_ptr<DelayPeakDetector> delay_peak_detector_ 345 GUARDED_BY(crit_sect_); 346 const scoped_ptr<DtmfBuffer> dtmf_buffer_ GUARDED_BY(crit_sect_); 347 const scoped_ptr<DtmfToneGenerator> dtmf_tone_generator_ 348 GUARDED_BY(crit_sect_); 349 const scoped_ptr<PacketBuffer> packet_buffer_ GUARDED_BY(crit_sect_); 350 const scoped_ptr<PayloadSplitter> payload_splitter_ GUARDED_BY(crit_sect_); 351 const scoped_ptr<TimestampScaler> timestamp_scaler_ GUARDED_BY(crit_sect_); 352 const scoped_ptr<PostDecodeVad> vad_ GUARDED_BY(crit_sect_); 353 const scoped_ptr<ExpandFactory> expand_factory_ GUARDED_BY(crit_sect_); 354 const scoped_ptr<AccelerateFactory> accelerate_factory_ 355 GUARDED_BY(crit_sect_); 356 const scoped_ptr<PreemptiveExpandFactory> preemptive_expand_factory_ 357 GUARDED_BY(crit_sect_); 358 359 scoped_ptr<BackgroundNoise> background_noise_ GUARDED_BY(crit_sect_); 360 scoped_ptr<DecisionLogic> decision_logic_ GUARDED_BY(crit_sect_); 361 scoped_ptr<AudioMultiVector> algorithm_buffer_ GUARDED_BY(crit_sect_); 362 scoped_ptr<SyncBuffer> sync_buffer_ GUARDED_BY(crit_sect_); 363 scoped_ptr<Expand> expand_ GUARDED_BY(crit_sect_); 364 scoped_ptr<Normal> normal_ GUARDED_BY(crit_sect_); 365 scoped_ptr<Merge> merge_ GUARDED_BY(crit_sect_); 366 scoped_ptr<Accelerate> accelerate_ GUARDED_BY(crit_sect_); 367 scoped_ptr<PreemptiveExpand> preemptive_expand_ GUARDED_BY(crit_sect_); 368 RandomVector random_vector_ GUARDED_BY(crit_sect_); 369 scoped_ptr<ComfortNoise> comfort_noise_ GUARDED_BY(crit_sect_); 370 Rtcp rtcp_ GUARDED_BY(crit_sect_); 371 StatisticsCalculator stats_ GUARDED_BY(crit_sect_); 372 int fs_hz_ GUARDED_BY(crit_sect_); 373 int fs_mult_ GUARDED_BY(crit_sect_); 374 int output_size_samples_ GUARDED_BY(crit_sect_); 375 int decoder_frame_length_ GUARDED_BY(crit_sect_); 376 Modes last_mode_ GUARDED_BY(crit_sect_); 377 scoped_ptr<int16_t[]> mute_factor_array_ GUARDED_BY(crit_sect_); 378 size_t decoded_buffer_length_ GUARDED_BY(crit_sect_); 379 scoped_ptr<int16_t[]> decoded_buffer_ GUARDED_BY(crit_sect_); 380 uint32_t playout_timestamp_ GUARDED_BY(crit_sect_); 381 bool new_codec_ GUARDED_BY(crit_sect_); 382 uint32_t timestamp_ GUARDED_BY(crit_sect_); 383 bool reset_decoder_ GUARDED_BY(crit_sect_); 384 uint8_t current_rtp_payload_type_ GUARDED_BY(crit_sect_); 385 uint8_t current_cng_rtp_payload_type_ GUARDED_BY(crit_sect_); 386 uint32_t ssrc_ GUARDED_BY(crit_sect_); 387 bool first_packet_ GUARDED_BY(crit_sect_); 388 int error_code_ GUARDED_BY(crit_sect_); // Store last error code. 389 int decoder_error_code_ GUARDED_BY(crit_sect_); 390 391 // These values are used by NACK module to estimate time-to-play of 392 // a missing packet. Occasionally, NetEq might decide to decode more 393 // than one packet. Therefore, these values store sequence number and 394 // timestamp of the first packet pulled from the packet buffer. In 395 // such cases, these values do not exactly represent the sequence number 396 // or timestamp associated with a 10ms audio pulled from NetEq. NACK 397 // module is designed to compensate for this. 398 int decoded_packet_sequence_number_ GUARDED_BY(crit_sect_); 399 uint32_t decoded_packet_timestamp_ GUARDED_BY(crit_sect_); 400 401 private: 402 DISALLOW_COPY_AND_ASSIGN(NetEqImpl); 403 }; 404 405 } // namespace webrtc 406 #endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ 407
