1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "media/cast/audio_receiver/audio_receiver.h" 6 7 #include "base/bind.h" 8 #include "base/logging.h" 9 #include "base/message_loop/message_loop.h" 10 #include "crypto/encryptor.h" 11 #include "crypto/symmetric_key.h" 12 #include "media/cast/audio_receiver/audio_decoder.h" 13 #include "media/cast/framer/framer.h" 14 #include "media/cast/rtcp/rtcp.h" 15 #include "media/cast/rtp_receiver/rtp_receiver.h" 16 17 // Max time we wait until an audio frame is due to be played out is released. 18 static const int64 kMaxAudioFrameWaitMs = 20; 19 static const int64 kMinSchedulingDelayMs = 1; 20 21 namespace media { 22 namespace cast { 23 24 DecodedAudioCallbackData::DecodedAudioCallbackData() 25 : number_of_10ms_blocks(0), 26 desired_frequency(0), 27 callback() {} 28 29 DecodedAudioCallbackData::~DecodedAudioCallbackData() {} 30 31 // Local implementation of RtpData (defined in rtp_rtcp_defines.h). 32 // Used to pass payload data into the audio receiver. 33 class LocalRtpAudioData : public RtpData { 34 public: 35 explicit LocalRtpAudioData(AudioReceiver* audio_receiver) 36 : audio_receiver_(audio_receiver) {} 37 38 virtual void OnReceivedPayloadData( 39 const uint8* payload_data, 40 size_t payload_size, 41 const RtpCastHeader* rtp_header) OVERRIDE { 42 audio_receiver_->IncomingParsedRtpPacket(payload_data, payload_size, 43 *rtp_header); 44 } 45 46 private: 47 AudioReceiver* audio_receiver_; 48 }; 49 50 // Local implementation of RtpPayloadFeedback (defined in rtp_defines.h) 51 // Used to convey cast-specific feedback from receiver to sender. 52 class LocalRtpAudioFeedback : public RtpPayloadFeedback { 53 public: 54 explicit LocalRtpAudioFeedback(AudioReceiver* audio_receiver) 55 : audio_receiver_(audio_receiver) { 56 } 57 58 virtual void CastFeedback(const RtcpCastMessage& cast_message) OVERRIDE { 59 audio_receiver_->CastFeedback(cast_message); 60 } 61 62 private: 63 AudioReceiver* audio_receiver_; 64 }; 65 66 class LocalRtpReceiverStatistics : public RtpReceiverStatistics { 67 public: 68 explicit LocalRtpReceiverStatistics(RtpReceiver* rtp_receiver) 69 : rtp_receiver_(rtp_receiver) { 70 } 71 72 virtual void GetStatistics(uint8* fraction_lost, 73 uint32* cumulative_lost, // 24 bits valid. 74 uint32* extended_high_sequence_number, 75 uint32* jitter) OVERRIDE { 76 rtp_receiver_->GetStatistics(fraction_lost, 77 cumulative_lost, 78 extended_high_sequence_number, 79 jitter); 80 } 81 82 private: 83 RtpReceiver* rtp_receiver_; 84 }; 85 86 AudioReceiver::AudioReceiver(scoped_refptr<CastEnvironment> cast_environment, 87 const AudioReceiverConfig& audio_config, 88 PacedPacketSender* const packet_sender) 89 : cast_environment_(cast_environment), 90 codec_(audio_config.codec), 91 frequency_(audio_config.frequency), 92 audio_buffer_(), 93 audio_decoder_(), 94 time_offset_(), 95 weak_factory_(this) { 96 target_delay_delta_ = 97 base::TimeDelta::FromMilliseconds(audio_config.rtp_max_delay_ms); 98 incoming_payload_callback_.reset(new LocalRtpAudioData(this)); 99 incoming_payload_feedback_.reset(new LocalRtpAudioFeedback(this)); 100 if (audio_config.use_external_decoder) { 101 audio_buffer_.reset(new Framer(cast_environment->Clock(), 102 incoming_payload_feedback_.get(), 103 audio_config.incoming_ssrc, 104 true, 105 0)); 106 } else { 107 audio_decoder_.reset(new AudioDecoder(cast_environment, 108 audio_config, 109 incoming_payload_feedback_.get())); 110 } 111 if (audio_config.aes_iv_mask.size() == kAesKeySize && 112 audio_config.aes_key.size() == kAesKeySize) { 113 iv_mask_ = audio_config.aes_iv_mask; 114 crypto::SymmetricKey* key = crypto::SymmetricKey::Import( 115 crypto::SymmetricKey::AES, audio_config.aes_key); 116 decryptor_.reset(new crypto::Encryptor()); 117 decryptor_->Init(key, crypto::Encryptor::CTR, std::string()); 118 } else if (audio_config.aes_iv_mask.size() != 0 || 119 audio_config.aes_key.size() != 0) { 120 DCHECK(false) << "Invalid crypto configuration"; 121 } 122 123 rtp_receiver_.reset(new RtpReceiver(cast_environment->Clock(), 124 &audio_config, 125 NULL, 126 incoming_payload_callback_.get())); 127 rtp_audio_receiver_statistics_.reset( 128 new LocalRtpReceiverStatistics(rtp_receiver_.get())); 129 base::TimeDelta rtcp_interval_delta = 130 base::TimeDelta::FromMilliseconds(audio_config.rtcp_interval); 131 rtcp_.reset(new Rtcp(cast_environment, 132 NULL, 133 packet_sender, 134 NULL, 135 rtp_audio_receiver_statistics_.get(), 136 audio_config.rtcp_mode, 137 rtcp_interval_delta, 138 audio_config.feedback_ssrc, 139 audio_config.incoming_ssrc, 140 audio_config.rtcp_c_name)); 141 } 142 143 AudioReceiver::~AudioReceiver() {} 144 145 void AudioReceiver::InitializeTimers() { 146 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 147 ScheduleNextRtcpReport(); 148 ScheduleNextCastMessage(); 149 } 150 151 void AudioReceiver::IncomingParsedRtpPacket(const uint8* payload_data, 152 size_t payload_size, 153 const RtpCastHeader& rtp_header) { 154 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 155 cast_environment_->Logging()->InsertPacketEvent(kPacketReceived, 156 rtp_header.webrtc.header.timestamp, rtp_header.frame_id, 157 rtp_header.packet_id, rtp_header.max_packet_id, payload_size); 158 159 // TODO(pwestin): update this as video to refresh over time. 160 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 161 if (time_first_incoming_packet_.is_null()) { 162 InitializeTimers(); 163 first_incoming_rtp_timestamp_ = rtp_header.webrtc.header.timestamp; 164 time_first_incoming_packet_ = cast_environment_->Clock()->NowTicks(); 165 } 166 167 if (audio_decoder_) { 168 DCHECK(!audio_buffer_) << "Invalid internal state"; 169 std::string plaintext(reinterpret_cast<const char*>(payload_data), 170 payload_size); 171 if (decryptor_) { 172 plaintext.clear(); 173 if (!decryptor_->SetCounter(GetAesNonce(rtp_header.frame_id, iv_mask_))) { 174 NOTREACHED() << "Failed to set counter"; 175 return; 176 } 177 if (!decryptor_->Decrypt(base::StringPiece(reinterpret_cast<const char*>( 178 payload_data), payload_size), &plaintext)) { 179 VLOG(0) << "Decryption error"; 180 return; 181 } 182 } 183 audio_decoder_->IncomingParsedRtpPacket( 184 reinterpret_cast<const uint8*>(plaintext.data()), plaintext.size(), 185 rtp_header); 186 if (!queued_decoded_callbacks_.empty()) { 187 DecodedAudioCallbackData decoded_data = queued_decoded_callbacks_.front(); 188 queued_decoded_callbacks_.pop_front(); 189 cast_environment_->PostTask(CastEnvironment::AUDIO_DECODER, FROM_HERE, 190 base::Bind(&AudioReceiver::DecodeAudioFrameThread, 191 base::Unretained(this), 192 decoded_data.number_of_10ms_blocks, 193 decoded_data.desired_frequency, 194 decoded_data.callback)); 195 } 196 return; 197 } 198 199 DCHECK(audio_buffer_) << "Invalid internal state"; 200 DCHECK(!audio_decoder_) << "Invalid internal state"; 201 202 bool complete = audio_buffer_->InsertPacket(payload_data, payload_size, 203 rtp_header); 204 if (!complete) return; // Audio frame not complete; wait for more packets. 205 if (queued_encoded_callbacks_.empty()) return; 206 AudioFrameEncodedCallback callback = queued_encoded_callbacks_.front(); 207 queued_encoded_callbacks_.pop_front(); 208 cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, 209 base::Bind(&AudioReceiver::GetEncodedAudioFrame, 210 weak_factory_.GetWeakPtr(), callback)); 211 } 212 213 void AudioReceiver::GetRawAudioFrame(int number_of_10ms_blocks, 214 int desired_frequency, const AudioFrameDecodedCallback& callback) { 215 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 216 DCHECK(audio_decoder_) << "Invalid function call in this configuration"; 217 // TODO(pwestin): we can skip this function by posting direct to the decoder. 218 cast_environment_->PostTask(CastEnvironment::AUDIO_DECODER, FROM_HERE, 219 base::Bind(&AudioReceiver::DecodeAudioFrameThread, 220 base::Unretained(this), 221 number_of_10ms_blocks, 222 desired_frequency, 223 callback)); 224 } 225 226 void AudioReceiver::DecodeAudioFrameThread( 227 int number_of_10ms_blocks, 228 int desired_frequency, 229 const AudioFrameDecodedCallback callback) { 230 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::AUDIO_DECODER)); 231 // TODO(mikhal): Allow the application to allocate this memory. 232 scoped_ptr<PcmAudioFrame> audio_frame(new PcmAudioFrame()); 233 234 uint32 rtp_timestamp = 0; 235 if (!audio_decoder_->GetRawAudioFrame(number_of_10ms_blocks, 236 desired_frequency, 237 audio_frame.get(), 238 &rtp_timestamp)) { 239 DecodedAudioCallbackData callback_data; 240 callback_data.number_of_10ms_blocks = number_of_10ms_blocks; 241 callback_data.desired_frequency = desired_frequency; 242 callback_data.callback = callback; 243 queued_decoded_callbacks_.push_back(callback_data); 244 return; 245 } 246 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); 247 248 cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, 249 base::Bind(&AudioReceiver::ReturnDecodedFrameWithPlayoutDelay, 250 base::Unretained(this), base::Passed(&audio_frame), rtp_timestamp, 251 callback)); 252 } 253 254 void AudioReceiver::ReturnDecodedFrameWithPlayoutDelay( 255 scoped_ptr<PcmAudioFrame> audio_frame, uint32 rtp_timestamp, 256 const AudioFrameDecodedCallback callback) { 257 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 258 cast_environment_->Logging()->InsertFrameEvent(kAudioFrameDecoded, 259 rtp_timestamp, kFrameIdUnknown); 260 261 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); 262 base::TimeTicks playout_time = GetPlayoutTime(now, rtp_timestamp); 263 264 cast_environment_->Logging()->InsertFrameEventWithDelay(kAudioPlayoutDelay, 265 rtp_timestamp, kFrameIdUnknown, playout_time - now); 266 267 // Frame is ready - Send back to the caller. 268 cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, 269 base::Bind(callback, base::Passed(&audio_frame), playout_time)); 270 } 271 272 void AudioReceiver::PlayoutTimeout() { 273 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 274 DCHECK(audio_buffer_) << "Invalid function call in this configuration"; 275 if (queued_encoded_callbacks_.empty()) { 276 // Already released by incoming packet. 277 return; 278 } 279 uint32 rtp_timestamp = 0; 280 bool next_frame = false; 281 scoped_ptr<EncodedAudioFrame> encoded_frame(new EncodedAudioFrame()); 282 283 if (!audio_buffer_->GetEncodedAudioFrame(encoded_frame.get(), 284 &rtp_timestamp, &next_frame)) { 285 // We have no audio frames. Wait for new packet(s). 286 // Since the application can post multiple AudioFrameEncodedCallback and 287 // we only check the next frame to play out we might have multiple timeout 288 // events firing after each other; however this should be a rare event. 289 VLOG(1) << "Failed to retrieved a complete frame at this point in time"; 290 return; 291 } 292 293 if (decryptor_ && !DecryptAudioFrame(&encoded_frame)) { 294 // Logging already done. 295 return; 296 } 297 298 if (PostEncodedAudioFrame(queued_encoded_callbacks_.front(), rtp_timestamp, 299 next_frame, &encoded_frame)) { 300 // Call succeed remove callback from list. 301 queued_encoded_callbacks_.pop_front(); 302 } 303 } 304 305 void AudioReceiver::GetEncodedAudioFrame( 306 const AudioFrameEncodedCallback& callback) { 307 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 308 DCHECK(audio_buffer_) << "Invalid function call in this configuration"; 309 310 uint32 rtp_timestamp = 0; 311 bool next_frame = false; 312 scoped_ptr<EncodedAudioFrame> encoded_frame(new EncodedAudioFrame()); 313 314 if (!audio_buffer_->GetEncodedAudioFrame(encoded_frame.get(), 315 &rtp_timestamp, &next_frame)) { 316 // We have no audio frames. Wait for new packet(s). 317 VLOG(1) << "Wait for more audio packets in frame"; 318 queued_encoded_callbacks_.push_back(callback); 319 return; 320 } 321 if (decryptor_ && !DecryptAudioFrame(&encoded_frame)) { 322 // Logging already done. 323 queued_encoded_callbacks_.push_back(callback); 324 return; 325 } 326 if (!PostEncodedAudioFrame(callback, rtp_timestamp, next_frame, 327 &encoded_frame)) { 328 // We have an audio frame; however we are missing packets and we have time 329 // to wait for new packet(s). 330 queued_encoded_callbacks_.push_back(callback); 331 } 332 } 333 334 bool AudioReceiver::PostEncodedAudioFrame( 335 const AudioFrameEncodedCallback& callback, 336 uint32 rtp_timestamp, 337 bool next_frame, 338 scoped_ptr<EncodedAudioFrame>* encoded_frame) { 339 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 340 DCHECK(audio_buffer_) << "Invalid function call in this configuration"; 341 342 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); 343 base::TimeTicks playout_time = GetPlayoutTime(now, rtp_timestamp); 344 base::TimeDelta time_until_playout = playout_time - now; 345 base::TimeDelta min_wait_delta = 346 base::TimeDelta::FromMilliseconds(kMaxAudioFrameWaitMs); 347 348 if (!next_frame && (time_until_playout > min_wait_delta)) { 349 base::TimeDelta time_until_release = time_until_playout - min_wait_delta; 350 cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE, 351 base::Bind(&AudioReceiver::PlayoutTimeout, weak_factory_.GetWeakPtr()), 352 time_until_release); 353 VLOG(1) << "Wait until time to playout:" 354 << time_until_release.InMilliseconds(); 355 return false; 356 } 357 (*encoded_frame)->codec = codec_; 358 audio_buffer_->ReleaseFrame((*encoded_frame)->frame_id); 359 360 cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, 361 base::Bind(callback, base::Passed(encoded_frame), playout_time)); 362 return true; 363 } 364 365 void AudioReceiver::IncomingPacket(const uint8* packet, size_t length, 366 const base::Closure callback) { 367 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 368 bool rtcp_packet = Rtcp::IsRtcpPacket(packet, length); 369 if (!rtcp_packet) { 370 rtp_receiver_->ReceivedPacket(packet, length); 371 } else { 372 rtcp_->IncomingRtcpPacket(packet, length); 373 } 374 cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback); 375 } 376 377 void AudioReceiver::CastFeedback(const RtcpCastMessage& cast_message) { 378 // TODO(pwestin): add logging. 379 rtcp_->SendRtcpFromRtpReceiver(&cast_message, NULL); 380 } 381 382 base::TimeTicks AudioReceiver::GetPlayoutTime(base::TimeTicks now, 383 uint32 rtp_timestamp) { 384 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 385 // Senders time in ms when this frame was recorded. 386 // Note: the senders clock and our local clock might not be synced. 387 base::TimeTicks rtp_timestamp_in_ticks; 388 if (time_offset_ == base::TimeDelta()) { 389 if (rtcp_->RtpTimestampInSenderTime(frequency_, 390 first_incoming_rtp_timestamp_, 391 &rtp_timestamp_in_ticks)) { 392 time_offset_ = time_first_incoming_packet_ - rtp_timestamp_in_ticks; 393 } else { 394 // We have not received any RTCP to sync the stream play it out as soon as 395 // possible. 396 uint32 rtp_timestamp_diff = rtp_timestamp - first_incoming_rtp_timestamp_; 397 398 int frequency_khz = frequency_ / 1000; 399 base::TimeDelta rtp_time_diff_delta = 400 base::TimeDelta::FromMilliseconds(rtp_timestamp_diff / frequency_khz); 401 base::TimeDelta time_diff_delta = now - time_first_incoming_packet_; 402 403 return now + std::max(rtp_time_diff_delta - time_diff_delta, 404 base::TimeDelta()); 405 } 406 } 407 // This can fail if we have not received any RTCP packets in a long time. 408 return rtcp_->RtpTimestampInSenderTime(frequency_, rtp_timestamp, 409 &rtp_timestamp_in_ticks) ? 410 rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_ : 411 now; 412 } 413 414 bool AudioReceiver::DecryptAudioFrame( 415 scoped_ptr<EncodedAudioFrame>* audio_frame) { 416 DCHECK(decryptor_) << "Invalid state"; 417 418 if (!decryptor_->SetCounter(GetAesNonce((*audio_frame)->frame_id, 419 iv_mask_))) { 420 NOTREACHED() << "Failed to set counter"; 421 return false; 422 } 423 std::string decrypted_audio_data; 424 if (!decryptor_->Decrypt((*audio_frame)->data, &decrypted_audio_data)) { 425 VLOG(0) << "Decryption error"; 426 // Give up on this frame, release it from jitter buffer. 427 audio_buffer_->ReleaseFrame((*audio_frame)->frame_id); 428 return false; 429 } 430 (*audio_frame)->data.swap(decrypted_audio_data); 431 return true; 432 } 433 434 void AudioReceiver::ScheduleNextRtcpReport() { 435 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 436 base::TimeDelta time_to_send = rtcp_->TimeToSendNextRtcpReport() - 437 cast_environment_->Clock()->NowTicks(); 438 439 time_to_send = std::max(time_to_send, 440 base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); 441 442 cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE, 443 base::Bind(&AudioReceiver::SendNextRtcpReport, 444 weak_factory_.GetWeakPtr()), time_to_send); 445 } 446 447 void AudioReceiver::SendNextRtcpReport() { 448 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 449 // TODO(pwestin): add logging. 450 rtcp_->SendRtcpFromRtpReceiver(NULL, NULL); 451 ScheduleNextRtcpReport(); 452 } 453 454 // Cast messages should be sent within a maximum interval. Schedule a call 455 // if not triggered elsewhere, e.g. by the cast message_builder. 456 void AudioReceiver::ScheduleNextCastMessage() { 457 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 458 base::TimeTicks send_time; 459 if (audio_buffer_) { 460 audio_buffer_->TimeToSendNextCastMessage(&send_time); 461 } else if (audio_decoder_) { 462 audio_decoder_->TimeToSendNextCastMessage(&send_time); 463 } else { 464 NOTREACHED(); 465 } 466 base::TimeDelta time_to_send = send_time - 467 cast_environment_->Clock()->NowTicks(); 468 time_to_send = std::max(time_to_send, 469 base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); 470 cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE, 471 base::Bind(&AudioReceiver::SendNextCastMessage, 472 weak_factory_.GetWeakPtr()), time_to_send); 473 } 474 475 void AudioReceiver::SendNextCastMessage() { 476 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 477 478 if (audio_buffer_) { 479 // Will only send a message if it is time. 480 audio_buffer_->SendCastMessage(); 481 } 482 if (audio_decoder_) { 483 // Will only send a message if it is time. 484 audio_decoder_->SendCastMessage(); 485 } 486 ScheduleNextCastMessage(); 487 } 488 489 } // namespace cast 490 } // namespace media 491
