1 /* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "webrtc/call/rampup_tests.h" 12 13 #include "testing/gtest/include/gtest/gtest.h" 14 #include "webrtc/base/checks.h" 15 #include "webrtc/base/platform_thread.h" 16 #include "webrtc/test/testsupport/perf_test.h" 17 18 namespace webrtc { 19 namespace { 20 21 static const int64_t kPollIntervalMs = 20; 22 23 std::vector<uint32_t> GenerateSsrcs(size_t num_streams, uint32_t ssrc_offset) { 24 std::vector<uint32_t> ssrcs; 25 for (size_t i = 0; i != num_streams; ++i) 26 ssrcs.push_back(static_cast<uint32_t>(ssrc_offset + i)); 27 return ssrcs; 28 } 29 } // namespace 30 31 RampUpTester::RampUpTester(size_t num_video_streams, 32 size_t num_audio_streams, 33 unsigned int start_bitrate_bps, 34 const std::string& extension_type, 35 bool rtx, 36 bool red) 37 : EndToEndTest(test::CallTest::kLongTimeoutMs), 38 event_(false, false), 39 clock_(Clock::GetRealTimeClock()), 40 num_video_streams_(num_video_streams), 41 num_audio_streams_(num_audio_streams), 42 rtx_(rtx), 43 red_(red), 44 send_stream_(nullptr), 45 start_bitrate_bps_(start_bitrate_bps), 46 start_bitrate_verified_(false), 47 expected_bitrate_bps_(0), 48 test_start_ms_(-1), 49 ramp_up_finished_ms_(-1), 50 extension_type_(extension_type), 51 video_ssrcs_(GenerateSsrcs(num_video_streams_, 100)), 52 video_rtx_ssrcs_(GenerateSsrcs(num_video_streams_, 200)), 53 audio_ssrcs_(GenerateSsrcs(num_audio_streams_, 300)), 54 poller_thread_(&BitrateStatsPollingThread, 55 this, 56 "BitrateStatsPollingThread"), 57 sender_call_(nullptr) { 58 EXPECT_LE(num_audio_streams_, 1u); 59 if (rtx_) { 60 for (size_t i = 0; i < video_ssrcs_.size(); ++i) 61 rtx_ssrc_map_[video_rtx_ssrcs_[i]] = video_ssrcs_[i]; 62 } 63 } 64 65 RampUpTester::~RampUpTester() { 66 event_.Set(); 67 } 68 69 Call::Config RampUpTester::GetSenderCallConfig() { 70 Call::Config call_config; 71 if (start_bitrate_bps_ != 0) { 72 call_config.bitrate_config.start_bitrate_bps = start_bitrate_bps_; 73 } 74 call_config.bitrate_config.min_bitrate_bps = 10000; 75 return call_config; 76 } 77 78 void RampUpTester::OnVideoStreamsCreated( 79 VideoSendStream* send_stream, 80 const std::vector<VideoReceiveStream*>& receive_streams) { 81 send_stream_ = send_stream; 82 } 83 84 test::PacketTransport* RampUpTester::CreateSendTransport(Call* sender_call) { 85 send_transport_ = new test::PacketTransport(sender_call, this, 86 test::PacketTransport::kSender, 87 forward_transport_config_); 88 return send_transport_; 89 } 90 91 size_t RampUpTester::GetNumVideoStreams() const { 92 return num_video_streams_; 93 } 94 95 size_t RampUpTester::GetNumAudioStreams() const { 96 return num_audio_streams_; 97 } 98 99 void RampUpTester::ModifyVideoConfigs( 100 VideoSendStream::Config* send_config, 101 std::vector<VideoReceiveStream::Config>* receive_configs, 102 VideoEncoderConfig* encoder_config) { 103 send_config->suspend_below_min_bitrate = true; 104 105 if (num_video_streams_ == 1) { 106 encoder_config->streams[0].target_bitrate_bps = 107 encoder_config->streams[0].max_bitrate_bps = 2000000; 108 // For single stream rampup until 1mbps 109 expected_bitrate_bps_ = kSingleStreamTargetBps; 110 } else { 111 // For multi stream rampup until all streams are being sent. That means 112 // enough birate to send all the target streams plus the min bitrate of 113 // the last one. 114 expected_bitrate_bps_ = encoder_config->streams.back().min_bitrate_bps; 115 for (size_t i = 0; i < encoder_config->streams.size() - 1; ++i) { 116 expected_bitrate_bps_ += encoder_config->streams[i].target_bitrate_bps; 117 } 118 } 119 120 send_config->rtp.extensions.clear(); 121 122 bool remb; 123 bool transport_cc; 124 if (extension_type_ == RtpExtension::kAbsSendTime) { 125 remb = true; 126 transport_cc = false; 127 send_config->rtp.extensions.push_back( 128 RtpExtension(extension_type_.c_str(), kAbsSendTimeExtensionId)); 129 } else if (extension_type_ == RtpExtension::kTransportSequenceNumber) { 130 remb = false; 131 transport_cc = true; 132 send_config->rtp.extensions.push_back(RtpExtension( 133 extension_type_.c_str(), kTransportSequenceNumberExtensionId)); 134 } else { 135 remb = true; 136 transport_cc = false; 137 send_config->rtp.extensions.push_back(RtpExtension( 138 extension_type_.c_str(), kTransmissionTimeOffsetExtensionId)); 139 } 140 141 send_config->rtp.nack.rtp_history_ms = test::CallTest::kNackRtpHistoryMs; 142 send_config->rtp.ssrcs = video_ssrcs_; 143 if (rtx_) { 144 send_config->rtp.rtx.payload_type = test::CallTest::kSendRtxPayloadType; 145 send_config->rtp.rtx.ssrcs = video_rtx_ssrcs_; 146 } 147 if (red_) { 148 send_config->rtp.fec.ulpfec_payload_type = 149 test::CallTest::kUlpfecPayloadType; 150 send_config->rtp.fec.red_payload_type = test::CallTest::kRedPayloadType; 151 } 152 153 size_t i = 0; 154 for (VideoReceiveStream::Config& recv_config : *receive_configs) { 155 recv_config.rtp.remb = remb; 156 recv_config.rtp.transport_cc = transport_cc; 157 recv_config.rtp.extensions = send_config->rtp.extensions; 158 159 recv_config.rtp.remote_ssrc = video_ssrcs_[i]; 160 recv_config.rtp.nack.rtp_history_ms = send_config->rtp.nack.rtp_history_ms; 161 162 if (red_) { 163 recv_config.rtp.fec.red_payload_type = 164 send_config->rtp.fec.red_payload_type; 165 recv_config.rtp.fec.ulpfec_payload_type = 166 send_config->rtp.fec.ulpfec_payload_type; 167 } 168 169 if (rtx_) { 170 recv_config.rtp.rtx[send_config->encoder_settings.payload_type].ssrc = 171 video_rtx_ssrcs_[i]; 172 recv_config.rtp.rtx[send_config->encoder_settings.payload_type] 173 .payload_type = send_config->rtp.rtx.payload_type; 174 } 175 ++i; 176 } 177 } 178 179 void RampUpTester::ModifyAudioConfigs( 180 AudioSendStream::Config* send_config, 181 std::vector<AudioReceiveStream::Config>* receive_configs) { 182 if (num_audio_streams_ == 0) 183 return; 184 185 EXPECT_NE(RtpExtension::kTOffset, extension_type_) 186 << "Audio BWE not supported with toffset."; 187 188 send_config->rtp.ssrc = audio_ssrcs_[0]; 189 send_config->rtp.extensions.clear(); 190 191 bool transport_cc = false; 192 if (extension_type_ == RtpExtension::kAbsSendTime) { 193 transport_cc = false; 194 send_config->rtp.extensions.push_back( 195 RtpExtension(extension_type_.c_str(), kAbsSendTimeExtensionId)); 196 } else if (extension_type_ == RtpExtension::kTransportSequenceNumber) { 197 transport_cc = true; 198 send_config->rtp.extensions.push_back(RtpExtension( 199 extension_type_.c_str(), kTransportSequenceNumberExtensionId)); 200 } 201 202 for (AudioReceiveStream::Config& recv_config : *receive_configs) { 203 recv_config.combined_audio_video_bwe = true; 204 recv_config.rtp.transport_cc = transport_cc; 205 recv_config.rtp.extensions = send_config->rtp.extensions; 206 recv_config.rtp.remote_ssrc = send_config->rtp.ssrc; 207 } 208 } 209 210 void RampUpTester::OnCallsCreated(Call* sender_call, Call* receiver_call) { 211 sender_call_ = sender_call; 212 } 213 214 bool RampUpTester::BitrateStatsPollingThread(void* obj) { 215 return static_cast<RampUpTester*>(obj)->PollStats(); 216 } 217 218 bool RampUpTester::PollStats() { 219 if (sender_call_) { 220 Call::Stats stats = sender_call_->GetStats(); 221 222 RTC_DCHECK_GT(expected_bitrate_bps_, 0); 223 if (!start_bitrate_verified_ && start_bitrate_bps_ != 0) { 224 // For tests with an explicitly set start bitrate, verify the first 225 // bitrate estimate is close to the start bitrate and lower than the 226 // test target bitrate. This is to verify a call respects the configured 227 // start bitrate, but due to the BWE implementation we can't guarantee the 228 // first estimate really is as high as the start bitrate. 229 EXPECT_GT(stats.send_bandwidth_bps, 0.9 * start_bitrate_bps_); 230 start_bitrate_verified_ = true; 231 } 232 if (stats.send_bandwidth_bps >= expected_bitrate_bps_) { 233 ramp_up_finished_ms_ = clock_->TimeInMilliseconds(); 234 observation_complete_.Set(); 235 } 236 } 237 238 return !event_.Wait(kPollIntervalMs); 239 } 240 241 void RampUpTester::ReportResult(const std::string& measurement, 242 size_t value, 243 const std::string& units) const { 244 webrtc::test::PrintResult( 245 measurement, "", 246 ::testing::UnitTest::GetInstance()->current_test_info()->name(), value, 247 units, false); 248 } 249 250 void RampUpTester::AccumulateStats(const VideoSendStream::StreamStats& stream, 251 size_t* total_packets_sent, 252 size_t* total_sent, 253 size_t* padding_sent, 254 size_t* media_sent) const { 255 *total_packets_sent += stream.rtp_stats.transmitted.packets + 256 stream.rtp_stats.retransmitted.packets + 257 stream.rtp_stats.fec.packets; 258 *total_sent += stream.rtp_stats.transmitted.TotalBytes() + 259 stream.rtp_stats.retransmitted.TotalBytes() + 260 stream.rtp_stats.fec.TotalBytes(); 261 *padding_sent += stream.rtp_stats.transmitted.padding_bytes + 262 stream.rtp_stats.retransmitted.padding_bytes + 263 stream.rtp_stats.fec.padding_bytes; 264 *media_sent += stream.rtp_stats.MediaPayloadBytes(); 265 } 266 267 void RampUpTester::TriggerTestDone() { 268 RTC_DCHECK_GE(test_start_ms_, 0); 269 270 // TODO(holmer): Add audio send stats here too when those APIs are available. 271 VideoSendStream::Stats send_stats = send_stream_->GetStats(); 272 273 size_t total_packets_sent = 0; 274 size_t total_sent = 0; 275 size_t padding_sent = 0; 276 size_t media_sent = 0; 277 for (uint32_t ssrc : video_ssrcs_) { 278 AccumulateStats(send_stats.substreams[ssrc], &total_packets_sent, 279 &total_sent, &padding_sent, &media_sent); 280 } 281 282 size_t rtx_total_packets_sent = 0; 283 size_t rtx_total_sent = 0; 284 size_t rtx_padding_sent = 0; 285 size_t rtx_media_sent = 0; 286 for (uint32_t rtx_ssrc : video_rtx_ssrcs_) { 287 AccumulateStats(send_stats.substreams[rtx_ssrc], &rtx_total_packets_sent, 288 &rtx_total_sent, &rtx_padding_sent, &rtx_media_sent); 289 } 290 291 ReportResult("ramp-up-total-packets-sent", total_packets_sent, "packets"); 292 ReportResult("ramp-up-total-sent", total_sent, "bytes"); 293 ReportResult("ramp-up-media-sent", media_sent, "bytes"); 294 ReportResult("ramp-up-padding-sent", padding_sent, "bytes"); 295 ReportResult("ramp-up-rtx-total-packets-sent", rtx_total_packets_sent, 296 "packets"); 297 ReportResult("ramp-up-rtx-total-sent", rtx_total_sent, "bytes"); 298 ReportResult("ramp-up-rtx-media-sent", rtx_media_sent, "bytes"); 299 ReportResult("ramp-up-rtx-padding-sent", rtx_padding_sent, "bytes"); 300 if (ramp_up_finished_ms_ >= 0) { 301 ReportResult("ramp-up-time", ramp_up_finished_ms_ - test_start_ms_, 302 "milliseconds"); 303 } 304 ReportResult("ramp-up-average-network-latency", 305 send_transport_->GetAverageDelayMs(), "milliseconds"); 306 } 307 308 void RampUpTester::PerformTest() { 309 test_start_ms_ = clock_->TimeInMilliseconds(); 310 poller_thread_.Start(); 311 EXPECT_TRUE(Wait()) << "Timed out while waiting for ramp-up to complete."; 312 TriggerTestDone(); 313 poller_thread_.Stop(); 314 } 315 316 RampUpDownUpTester::RampUpDownUpTester(size_t num_video_streams, 317 size_t num_audio_streams, 318 unsigned int start_bitrate_bps, 319 const std::string& extension_type, 320 bool rtx, 321 bool red) 322 : RampUpTester(num_video_streams, 323 num_audio_streams, 324 start_bitrate_bps, 325 extension_type, 326 rtx, 327 red), 328 test_state_(kFirstRampup), 329 state_start_ms_(clock_->TimeInMilliseconds()), 330 interval_start_ms_(clock_->TimeInMilliseconds()), 331 sent_bytes_(0) { 332 forward_transport_config_.link_capacity_kbps = kHighBandwidthLimitBps / 1000; 333 } 334 335 RampUpDownUpTester::~RampUpDownUpTester() {} 336 337 bool RampUpDownUpTester::PollStats() { 338 if (send_stream_) { 339 webrtc::VideoSendStream::Stats stats = send_stream_->GetStats(); 340 int transmit_bitrate_bps = 0; 341 for (auto it : stats.substreams) { 342 transmit_bitrate_bps += it.second.total_bitrate_bps; 343 } 344 345 EvolveTestState(transmit_bitrate_bps, stats.suspended); 346 } 347 348 return !event_.Wait(kPollIntervalMs); 349 } 350 351 Call::Config RampUpDownUpTester::GetReceiverCallConfig() { 352 Call::Config config; 353 config.bitrate_config.min_bitrate_bps = 10000; 354 return config; 355 } 356 357 std::string RampUpDownUpTester::GetModifierString() const { 358 std::string str("_"); 359 if (num_video_streams_ > 0) { 360 std::ostringstream s; 361 s << num_video_streams_; 362 str += s.str(); 363 str += "stream"; 364 str += (num_video_streams_ > 1 ? "s" : ""); 365 str += "_"; 366 } 367 if (num_audio_streams_ > 0) { 368 std::ostringstream s; 369 s << num_audio_streams_; 370 str += s.str(); 371 str += "stream"; 372 str += (num_audio_streams_ > 1 ? "s" : ""); 373 str += "_"; 374 } 375 str += (rtx_ ? "" : "no"); 376 str += "rtx"; 377 return str; 378 } 379 380 void RampUpDownUpTester::EvolveTestState(int bitrate_bps, bool suspended) { 381 int64_t now = clock_->TimeInMilliseconds(); 382 switch (test_state_) { 383 case kFirstRampup: { 384 EXPECT_FALSE(suspended); 385 if (bitrate_bps > kExpectedHighBitrateBps) { 386 // The first ramp-up has reached the target bitrate. Change the 387 // channel limit, and move to the next test state. 388 forward_transport_config_.link_capacity_kbps = 389 kLowBandwidthLimitBps / 1000; 390 send_transport_->SetConfig(forward_transport_config_); 391 test_state_ = kLowRate; 392 webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(), 393 "first_rampup", now - state_start_ms_, "ms", 394 false); 395 state_start_ms_ = now; 396 interval_start_ms_ = now; 397 sent_bytes_ = 0; 398 } 399 break; 400 } 401 case kLowRate: { 402 if (bitrate_bps < kExpectedLowBitrateBps && suspended) { 403 // The ramp-down was successful. Change the channel limit back to a 404 // high value, and move to the next test state. 405 forward_transport_config_.link_capacity_kbps = 406 kHighBandwidthLimitBps / 1000; 407 send_transport_->SetConfig(forward_transport_config_); 408 test_state_ = kSecondRampup; 409 webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(), 410 "rampdown", now - state_start_ms_, "ms", 411 false); 412 state_start_ms_ = now; 413 interval_start_ms_ = now; 414 sent_bytes_ = 0; 415 } 416 break; 417 } 418 case kSecondRampup: { 419 if (bitrate_bps > kExpectedHighBitrateBps && !suspended) { 420 webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(), 421 "second_rampup", now - state_start_ms_, "ms", 422 false); 423 ReportResult("ramp-up-down-up-average-network-latency", 424 send_transport_->GetAverageDelayMs(), "milliseconds"); 425 observation_complete_.Set(); 426 } 427 break; 428 } 429 } 430 } 431 432 class RampUpTest : public test::CallTest { 433 public: 434 RampUpTest() {} 435 436 virtual ~RampUpTest() { 437 EXPECT_EQ(nullptr, video_send_stream_); 438 EXPECT_TRUE(video_receive_streams_.empty()); 439 } 440 }; 441 442 TEST_F(RampUpTest, SingleStream) { 443 RampUpTester test(1, 0, 0, RtpExtension::kTOffset, false, false); 444 RunBaseTest(&test); 445 } 446 447 TEST_F(RampUpTest, Simulcast) { 448 RampUpTester test(3, 0, 0, RtpExtension::kTOffset, false, false); 449 RunBaseTest(&test); 450 } 451 452 TEST_F(RampUpTest, SimulcastWithRtx) { 453 RampUpTester test(3, 0, 0, RtpExtension::kTOffset, true, false); 454 RunBaseTest(&test); 455 } 456 457 TEST_F(RampUpTest, SimulcastByRedWithRtx) { 458 RampUpTester test(3, 0, 0, RtpExtension::kTOffset, true, true); 459 RunBaseTest(&test); 460 } 461 462 TEST_F(RampUpTest, SingleStreamWithHighStartBitrate) { 463 RampUpTester test(1, 0, 0.9 * kSingleStreamTargetBps, RtpExtension::kTOffset, 464 false, false); 465 RunBaseTest(&test); 466 } 467 468 // Disabled on Mac due to flakiness, see 469 // https://bugs.chromium.org/p/webrtc/issues/detail?id=5407 470 #ifndef WEBRTC_MAC 471 472 static const uint32_t kStartBitrateBps = 60000; 473 474 TEST_F(RampUpTest, UpDownUpOneStream) { 475 RampUpDownUpTester test(1, 0, kStartBitrateBps, RtpExtension::kAbsSendTime, 476 false, false); 477 RunBaseTest(&test); 478 } 479 480 TEST_F(RampUpTest, UpDownUpThreeStreams) { 481 RampUpDownUpTester test(3, 0, kStartBitrateBps, RtpExtension::kAbsSendTime, 482 false, false); 483 RunBaseTest(&test); 484 } 485 486 TEST_F(RampUpTest, UpDownUpOneStreamRtx) { 487 RampUpDownUpTester test(1, 0, kStartBitrateBps, RtpExtension::kAbsSendTime, 488 true, false); 489 RunBaseTest(&test); 490 } 491 492 TEST_F(RampUpTest, UpDownUpThreeStreamsRtx) { 493 RampUpDownUpTester test(3, 0, kStartBitrateBps, RtpExtension::kAbsSendTime, 494 true, false); 495 RunBaseTest(&test); 496 } 497 498 TEST_F(RampUpTest, UpDownUpOneStreamByRedRtx) { 499 RampUpDownUpTester test(1, 0, kStartBitrateBps, RtpExtension::kAbsSendTime, 500 true, true); 501 RunBaseTest(&test); 502 } 503 504 TEST_F(RampUpTest, UpDownUpThreeStreamsByRedRtx) { 505 RampUpDownUpTester test(3, 0, kStartBitrateBps, RtpExtension::kAbsSendTime, 506 true, true); 507 RunBaseTest(&test); 508 } 509 510 TEST_F(RampUpTest, SendSideVideoUpDownUpRtx) { 511 RampUpDownUpTester test(3, 0, kStartBitrateBps, 512 RtpExtension::kTransportSequenceNumber, true, false); 513 RunBaseTest(&test); 514 } 515 516 // TODO(holmer): Enable when audio bitrates are included in the bitrate 517 // allocation. 518 TEST_F(RampUpTest, DISABLED_SendSideAudioVideoUpDownUpRtx) { 519 RampUpDownUpTester test(3, 1, kStartBitrateBps, 520 RtpExtension::kTransportSequenceNumber, true, false); 521 RunBaseTest(&test); 522 } 523 524 #endif 525 526 TEST_F(RampUpTest, AbsSendTimeSingleStream) { 527 RampUpTester test(1, 0, 0, RtpExtension::kAbsSendTime, false, false); 528 RunBaseTest(&test); 529 } 530 531 TEST_F(RampUpTest, AbsSendTimeSimulcast) { 532 RampUpTester test(3, 0, 0, RtpExtension::kAbsSendTime, false, false); 533 RunBaseTest(&test); 534 } 535 536 TEST_F(RampUpTest, AbsSendTimeSimulcastWithRtx) { 537 RampUpTester test(3, 0, 0, RtpExtension::kAbsSendTime, true, false); 538 RunBaseTest(&test); 539 } 540 541 TEST_F(RampUpTest, AbsSendTimeSimulcastByRedWithRtx) { 542 RampUpTester test(3, 0, 0, RtpExtension::kAbsSendTime, true, true); 543 RunBaseTest(&test); 544 } 545 546 TEST_F(RampUpTest, AbsSendTimeSingleStreamWithHighStartBitrate) { 547 RampUpTester test(1, 0, 0.9 * kSingleStreamTargetBps, 548 RtpExtension::kAbsSendTime, false, false); 549 RunBaseTest(&test); 550 } 551 552 TEST_F(RampUpTest, TransportSequenceNumberSingleStream) { 553 RampUpTester test(1, 0, 0, RtpExtension::kTransportSequenceNumber, false, 554 false); 555 RunBaseTest(&test); 556 } 557 558 TEST_F(RampUpTest, TransportSequenceNumberSimulcast) { 559 RampUpTester test(3, 0, 0, RtpExtension::kTransportSequenceNumber, false, 560 false); 561 RunBaseTest(&test); 562 } 563 564 TEST_F(RampUpTest, TransportSequenceNumberSimulcastWithRtx) { 565 RampUpTester test(3, 0, 0, RtpExtension::kTransportSequenceNumber, true, 566 false); 567 RunBaseTest(&test); 568 } 569 570 TEST_F(RampUpTest, AudioVideoTransportSequenceNumberSimulcastWithRtx) { 571 RampUpTester test(3, 1, 0, RtpExtension::kTransportSequenceNumber, true, 572 false); 573 RunBaseTest(&test); 574 } 575 576 TEST_F(RampUpTest, TransportSequenceNumberSimulcastByRedWithRtx) { 577 RampUpTester test(3, 0, 0, RtpExtension::kTransportSequenceNumber, true, 578 true); 579 RunBaseTest(&test); 580 } 581 582 TEST_F(RampUpTest, TransportSequenceNumberSingleStreamWithHighStartBitrate) { 583 RampUpTester test(1, 0, 0.9 * kSingleStreamTargetBps, 584 RtpExtension::kTransportSequenceNumber, false, false); 585 RunBaseTest(&test); 586 } 587 } // namespace webrtc 588