1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "media/cast/audio_sender/audio_sender.h" 6 7 #include "base/bind.h" 8 #include "base/logging.h" 9 #include "base/message_loop/message_loop.h" 10 #include "media/cast/audio_sender/audio_encoder.h" 11 #include "media/cast/cast_defines.h" 12 #include "media/cast/rtcp/rtcp_defines.h" 13 #include "media/cast/transport/cast_transport_config.h" 14 15 namespace media { 16 namespace cast { 17 namespace { 18 19 const int kNumAggressiveReportsSentAtStart = 100; 20 const int kMinSchedulingDelayMs = 1; 21 22 // TODO(miu): This should be specified in AudioSenderConfig, but currently it is 23 // fixed to 100 FPS (i.e., 10 ms per frame), and AudioEncoder assumes this as 24 // well. 25 const int kAudioFrameRate = 100; 26 27 // Helper function to compute the maximum unacked audio frames that is sent. 28 int GetMaxUnackedFrames(base::TimeDelta target_delay) { 29 // As long as it doesn't go over |kMaxUnackedFrames|, it is okay to send more 30 // audio data than the target delay would suggest. Audio packets are tiny and 31 // receiver has the ability to drop any one of the packets. 32 // We send up to three times of the target delay of audio frames. 33 int frames = 34 1 + 3 * target_delay * kAudioFrameRate / base::TimeDelta::FromSeconds(1); 35 return std::min(kMaxUnackedFrames, frames); 36 } 37 } // namespace 38 39 AudioSender::AudioSender(scoped_refptr<CastEnvironment> cast_environment, 40 const AudioSenderConfig& audio_config, 41 transport::CastTransportSender* const transport_sender) 42 : cast_environment_(cast_environment), 43 target_playout_delay_(base::TimeDelta::FromMilliseconds( 44 audio_config.rtp_config.max_delay_ms)), 45 transport_sender_(transport_sender), 46 max_unacked_frames_(GetMaxUnackedFrames(target_playout_delay_)), 47 configured_encoder_bitrate_(audio_config.bitrate), 48 rtcp_(cast_environment, 49 this, 50 transport_sender_, 51 NULL, // paced sender. 52 NULL, 53 audio_config.rtcp_mode, 54 base::TimeDelta::FromMilliseconds(audio_config.rtcp_interval), 55 audio_config.rtp_config.ssrc, 56 audio_config.incoming_feedback_ssrc, 57 audio_config.rtcp_c_name, 58 AUDIO_EVENT), 59 rtp_timestamp_helper_(audio_config.frequency), 60 num_aggressive_rtcp_reports_sent_(0), 61 last_sent_frame_id_(0), 62 latest_acked_frame_id_(0), 63 duplicate_ack_counter_(0), 64 cast_initialization_status_(STATUS_AUDIO_UNINITIALIZED), 65 weak_factory_(this) { 66 VLOG(1) << "max_unacked_frames " << max_unacked_frames_; 67 DCHECK_GT(max_unacked_frames_, 0); 68 69 if (!audio_config.use_external_encoder) { 70 audio_encoder_.reset( 71 new AudioEncoder(cast_environment, 72 audio_config, 73 base::Bind(&AudioSender::SendEncodedAudioFrame, 74 weak_factory_.GetWeakPtr()))); 75 cast_initialization_status_ = audio_encoder_->InitializationResult(); 76 } else { 77 NOTREACHED(); // No support for external audio encoding. 78 cast_initialization_status_ = STATUS_AUDIO_UNINITIALIZED; 79 } 80 81 media::cast::transport::CastTransportAudioConfig transport_config; 82 transport_config.codec = audio_config.codec; 83 transport_config.rtp.config = audio_config.rtp_config; 84 transport_config.frequency = audio_config.frequency; 85 transport_config.channels = audio_config.channels; 86 transport_config.rtp.max_outstanding_frames = max_unacked_frames_; 87 transport_sender_->InitializeAudio(transport_config); 88 89 rtcp_.SetCastReceiverEventHistorySize(kReceiverRtcpEventHistorySize); 90 91 memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_)); 92 } 93 94 AudioSender::~AudioSender() {} 95 96 void AudioSender::InsertAudio(scoped_ptr<AudioBus> audio_bus, 97 const base::TimeTicks& recorded_time) { 98 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 99 if (cast_initialization_status_ != STATUS_AUDIO_INITIALIZED) { 100 NOTREACHED(); 101 return; 102 } 103 DCHECK(audio_encoder_.get()) << "Invalid internal state"; 104 105 if (AreTooManyFramesInFlight()) { 106 VLOG(1) << "Dropping frame due to too many frames currently in-flight."; 107 return; 108 } 109 110 audio_encoder_->InsertAudio(audio_bus.Pass(), recorded_time); 111 } 112 113 void AudioSender::SendEncodedAudioFrame( 114 scoped_ptr<transport::EncodedFrame> encoded_frame) { 115 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 116 117 const uint32 frame_id = encoded_frame->frame_id; 118 119 const bool is_first_frame_to_be_sent = last_send_time_.is_null(); 120 last_send_time_ = cast_environment_->Clock()->NowTicks(); 121 last_sent_frame_id_ = frame_id; 122 // If this is the first frame about to be sent, fake the value of 123 // |latest_acked_frame_id_| to indicate the receiver starts out all caught up. 124 // Also, schedule the periodic frame re-send checks. 125 if (is_first_frame_to_be_sent) { 126 latest_acked_frame_id_ = frame_id - 1; 127 ScheduleNextResendCheck(); 128 } 129 130 cast_environment_->Logging()->InsertEncodedFrameEvent( 131 last_send_time_, FRAME_ENCODED, AUDIO_EVENT, encoded_frame->rtp_timestamp, 132 frame_id, static_cast<int>(encoded_frame->data.size()), 133 encoded_frame->dependency == transport::EncodedFrame::KEY, 134 configured_encoder_bitrate_); 135 // Only use lowest 8 bits as key. 136 frame_id_to_rtp_timestamp_[frame_id & 0xff] = encoded_frame->rtp_timestamp; 137 138 DCHECK(!encoded_frame->reference_time.is_null()); 139 rtp_timestamp_helper_.StoreLatestTime(encoded_frame->reference_time, 140 encoded_frame->rtp_timestamp); 141 142 // At the start of the session, it's important to send reports before each 143 // frame so that the receiver can properly compute playout times. The reason 144 // more than one report is sent is because transmission is not guaranteed, 145 // only best effort, so we send enough that one should almost certainly get 146 // through. 147 if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) { 148 // SendRtcpReport() will schedule future reports to be made if this is the 149 // last "aggressive report." 150 ++num_aggressive_rtcp_reports_sent_; 151 const bool is_last_aggressive_report = 152 (num_aggressive_rtcp_reports_sent_ == kNumAggressiveReportsSentAtStart); 153 VLOG_IF(1, is_last_aggressive_report) << "Sending last aggressive report."; 154 SendRtcpReport(is_last_aggressive_report); 155 } 156 157 transport_sender_->InsertCodedAudioFrame(*encoded_frame); 158 } 159 160 void AudioSender::IncomingRtcpPacket(scoped_ptr<Packet> packet) { 161 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 162 rtcp_.IncomingRtcpPacket(&packet->front(), packet->size()); 163 } 164 165 void AudioSender::ScheduleNextRtcpReport() { 166 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 167 base::TimeDelta time_to_next = 168 rtcp_.TimeToSendNextRtcpReport() - cast_environment_->Clock()->NowTicks(); 169 170 time_to_next = std::max( 171 time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); 172 173 cast_environment_->PostDelayedTask( 174 CastEnvironment::MAIN, 175 FROM_HERE, 176 base::Bind(&AudioSender::SendRtcpReport, 177 weak_factory_.GetWeakPtr(), 178 true), 179 time_to_next); 180 } 181 182 void AudioSender::SendRtcpReport(bool schedule_future_reports) { 183 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 184 const base::TimeTicks now = cast_environment_->Clock()->NowTicks(); 185 uint32 now_as_rtp_timestamp = 0; 186 if (rtp_timestamp_helper_.GetCurrentTimeAsRtpTimestamp( 187 now, &now_as_rtp_timestamp)) { 188 rtcp_.SendRtcpFromRtpSender(now, now_as_rtp_timestamp); 189 } else { 190 // |rtp_timestamp_helper_| should have stored a mapping by this point. 191 NOTREACHED(); 192 } 193 if (schedule_future_reports) 194 ScheduleNextRtcpReport(); 195 } 196 197 void AudioSender::ScheduleNextResendCheck() { 198 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 199 DCHECK(!last_send_time_.is_null()); 200 base::TimeDelta time_to_next = 201 last_send_time_ - cast_environment_->Clock()->NowTicks() + 202 target_playout_delay_; 203 time_to_next = std::max( 204 time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); 205 cast_environment_->PostDelayedTask( 206 CastEnvironment::MAIN, 207 FROM_HERE, 208 base::Bind(&AudioSender::ResendCheck, weak_factory_.GetWeakPtr()), 209 time_to_next); 210 } 211 212 void AudioSender::ResendCheck() { 213 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 214 DCHECK(!last_send_time_.is_null()); 215 const base::TimeDelta time_since_last_send = 216 cast_environment_->Clock()->NowTicks() - last_send_time_; 217 if (time_since_last_send > target_playout_delay_) { 218 if (latest_acked_frame_id_ == last_sent_frame_id_) { 219 // Last frame acked, no point in doing anything 220 } else { 221 VLOG(1) << "ACK timeout; last acked frame: " << latest_acked_frame_id_; 222 ResendForKickstart(); 223 } 224 } 225 ScheduleNextResendCheck(); 226 } 227 228 void AudioSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) { 229 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 230 231 if (rtcp_.is_rtt_available()) { 232 // Having the RTT values implies the receiver sent back a receiver report 233 // based on it having received a report from here. Therefore, ensure this 234 // sender stops aggressively sending reports. 235 if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) { 236 VLOG(1) << "No longer a need to send reports aggressively (sent " 237 << num_aggressive_rtcp_reports_sent_ << ")."; 238 num_aggressive_rtcp_reports_sent_ = kNumAggressiveReportsSentAtStart; 239 ScheduleNextRtcpReport(); 240 } 241 } 242 243 if (last_send_time_.is_null()) 244 return; // Cannot get an ACK without having first sent a frame. 245 246 if (cast_feedback.missing_frames_and_packets_.empty()) { 247 // We only count duplicate ACKs when we have sent newer frames. 248 if (latest_acked_frame_id_ == cast_feedback.ack_frame_id_ && 249 latest_acked_frame_id_ != last_sent_frame_id_) { 250 duplicate_ack_counter_++; 251 } else { 252 duplicate_ack_counter_ = 0; 253 } 254 // TODO(miu): The values "2" and "3" should be derived from configuration. 255 if (duplicate_ack_counter_ >= 2 && duplicate_ack_counter_ % 3 == 2) { 256 VLOG(1) << "Received duplicate ACK for frame " << latest_acked_frame_id_; 257 ResendForKickstart(); 258 } 259 } else { 260 // Only count duplicated ACKs if there is no NACK request in between. 261 // This is to avoid aggresive resend. 262 duplicate_ack_counter_ = 0; 263 264 base::TimeDelta rtt; 265 base::TimeDelta avg_rtt; 266 base::TimeDelta min_rtt; 267 base::TimeDelta max_rtt; 268 rtcp_.Rtt(&rtt, &avg_rtt, &min_rtt, &max_rtt); 269 270 // A NACK is also used to cancel pending re-transmissions. 271 transport_sender_->ResendPackets( 272 true, cast_feedback.missing_frames_and_packets_, false, min_rtt); 273 } 274 275 const base::TimeTicks now = cast_environment_->Clock()->NowTicks(); 276 277 const RtpTimestamp rtp_timestamp = 278 frame_id_to_rtp_timestamp_[cast_feedback.ack_frame_id_ & 0xff]; 279 cast_environment_->Logging()->InsertFrameEvent(now, 280 FRAME_ACK_RECEIVED, 281 AUDIO_EVENT, 282 rtp_timestamp, 283 cast_feedback.ack_frame_id_); 284 285 const bool is_acked_out_of_order = 286 static_cast<int32>(cast_feedback.ack_frame_id_ - 287 latest_acked_frame_id_) < 0; 288 VLOG(2) << "Received ACK" << (is_acked_out_of_order ? " out-of-order" : "") 289 << " for frame " << cast_feedback.ack_frame_id_; 290 if (!is_acked_out_of_order) { 291 // Cancel resends of acked frames. 292 MissingFramesAndPacketsMap missing_frames_and_packets; 293 PacketIdSet missing; 294 while (latest_acked_frame_id_ != cast_feedback.ack_frame_id_) { 295 latest_acked_frame_id_++; 296 missing_frames_and_packets[latest_acked_frame_id_] = missing; 297 } 298 transport_sender_->ResendPackets( 299 true, missing_frames_and_packets, true, base::TimeDelta()); 300 latest_acked_frame_id_ = cast_feedback.ack_frame_id_; 301 } 302 } 303 304 bool AudioSender::AreTooManyFramesInFlight() const { 305 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 306 int frames_in_flight = 0; 307 if (!last_send_time_.is_null()) { 308 frames_in_flight += 309 static_cast<int32>(last_sent_frame_id_ - latest_acked_frame_id_); 310 } 311 VLOG(2) << frames_in_flight 312 << " frames in flight; last sent: " << last_sent_frame_id_ 313 << " latest acked: " << latest_acked_frame_id_; 314 return frames_in_flight >= max_unacked_frames_; 315 } 316 317 void AudioSender::ResendForKickstart() { 318 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); 319 DCHECK(!last_send_time_.is_null()); 320 VLOG(1) << "Resending last packet of frame " << last_sent_frame_id_ 321 << " to kick-start."; 322 // Send the first packet of the last encoded frame to kick start 323 // retransmission. This gives enough information to the receiver what 324 // packets and frames are missing. 325 MissingFramesAndPacketsMap missing_frames_and_packets; 326 PacketIdSet missing; 327 missing.insert(kRtcpCastLastPacket); 328 missing_frames_and_packets.insert( 329 std::make_pair(last_sent_frame_id_, missing)); 330 last_send_time_ = cast_environment_->Clock()->NowTicks(); 331 332 base::TimeDelta rtt; 333 base::TimeDelta avg_rtt; 334 base::TimeDelta min_rtt; 335 base::TimeDelta max_rtt; 336 rtcp_.Rtt(&rtt, &avg_rtt, &min_rtt, &max_rtt); 337 338 // Sending this extra packet is to kick-start the session. There is 339 // no need to optimize re-transmission for this case. 340 transport_sender_->ResendPackets( 341 true, missing_frames_and_packets, false, min_rtt); 342 } 343 344 } // namespace cast 345 } // namespace media 346
