1 /* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 #include <assert.h> 11 12 #include <algorithm> 13 #include <sstream> 14 #include <string> 15 16 #include "testing/gtest/include/gtest/gtest.h" 17 18 #include "webrtc/base/thread_annotations.h" 19 #include "webrtc/call.h" 20 #include "webrtc/modules/audio_coding/main/interface/audio_coding_module.h" 21 #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" 22 #include "webrtc/modules/rtp_rtcp/source/rtcp_utility.h" 23 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" 24 #include "webrtc/system_wrappers/interface/rtp_to_ntp.h" 25 #include "webrtc/system_wrappers/interface/scoped_ptr.h" 26 #include "webrtc/test/call_test.h" 27 #include "webrtc/test/direct_transport.h" 28 #include "webrtc/test/encoder_settings.h" 29 #include "webrtc/test/fake_audio_device.h" 30 #include "webrtc/test/fake_decoder.h" 31 #include "webrtc/test/fake_encoder.h" 32 #include "webrtc/test/frame_generator.h" 33 #include "webrtc/test/frame_generator_capturer.h" 34 #include "webrtc/test/rtp_rtcp_observer.h" 35 #include "webrtc/test/testsupport/fileutils.h" 36 #include "webrtc/test/testsupport/perf_test.h" 37 #include "webrtc/video/transport_adapter.h" 38 #include "webrtc/voice_engine/include/voe_base.h" 39 #include "webrtc/voice_engine/include/voe_codec.h" 40 #include "webrtc/voice_engine/include/voe_network.h" 41 #include "webrtc/voice_engine/include/voe_rtp_rtcp.h" 42 #include "webrtc/voice_engine/include/voe_video_sync.h" 43 44 namespace webrtc { 45 46 class CallPerfTest : public test::CallTest { 47 protected: 48 void TestAudioVideoSync(bool fec); 49 50 void TestMinTransmitBitrate(bool pad_to_min_bitrate); 51 52 void TestCaptureNtpTime(const FakeNetworkPipe::Config& net_config, 53 int threshold_ms, 54 int start_time_ms, 55 int run_time_ms); 56 }; 57 58 class SyncRtcpObserver : public test::RtpRtcpObserver { 59 public: 60 explicit SyncRtcpObserver(const FakeNetworkPipe::Config& config) 61 : test::RtpRtcpObserver(CallPerfTest::kLongTimeoutMs, config), 62 crit_(CriticalSectionWrapper::CreateCriticalSection()) {} 63 64 virtual Action OnSendRtcp(const uint8_t* packet, size_t length) OVERRIDE { 65 RTCPUtility::RTCPParserV2 parser(packet, length, true); 66 EXPECT_TRUE(parser.IsValid()); 67 68 for (RTCPUtility::RTCPPacketTypes packet_type = parser.Begin(); 69 packet_type != RTCPUtility::kRtcpNotValidCode; 70 packet_type = parser.Iterate()) { 71 if (packet_type == RTCPUtility::kRtcpSrCode) { 72 const RTCPUtility::RTCPPacket& packet = parser.Packet(); 73 RtcpMeasurement ntp_rtp_pair( 74 packet.SR.NTPMostSignificant, 75 packet.SR.NTPLeastSignificant, 76 packet.SR.RTPTimestamp); 77 StoreNtpRtpPair(ntp_rtp_pair); 78 } 79 } 80 return SEND_PACKET; 81 } 82 83 int64_t RtpTimestampToNtp(uint32_t timestamp) const { 84 CriticalSectionScoped lock(crit_.get()); 85 int64_t timestamp_in_ms = -1; 86 if (ntp_rtp_pairs_.size() == 2) { 87 // TODO(stefan): We can't EXPECT_TRUE on this call due to a bug in the 88 // RTCP sender where it sends RTCP SR before any RTP packets, which leads 89 // to a bogus NTP/RTP mapping. 90 RtpToNtpMs(timestamp, ntp_rtp_pairs_, ×tamp_in_ms); 91 return timestamp_in_ms; 92 } 93 return -1; 94 } 95 96 private: 97 void StoreNtpRtpPair(RtcpMeasurement ntp_rtp_pair) { 98 CriticalSectionScoped lock(crit_.get()); 99 for (RtcpList::iterator it = ntp_rtp_pairs_.begin(); 100 it != ntp_rtp_pairs_.end(); 101 ++it) { 102 if (ntp_rtp_pair.ntp_secs == it->ntp_secs && 103 ntp_rtp_pair.ntp_frac == it->ntp_frac) { 104 // This RTCP has already been added to the list. 105 return; 106 } 107 } 108 // We need two RTCP SR reports to map between RTP and NTP. More than two 109 // will not improve the mapping. 110 if (ntp_rtp_pairs_.size() == 2) { 111 ntp_rtp_pairs_.pop_back(); 112 } 113 ntp_rtp_pairs_.push_front(ntp_rtp_pair); 114 } 115 116 const scoped_ptr<CriticalSectionWrapper> crit_; 117 RtcpList ntp_rtp_pairs_ GUARDED_BY(crit_); 118 }; 119 120 class VideoRtcpAndSyncObserver : public SyncRtcpObserver, public VideoRenderer { 121 static const int kInSyncThresholdMs = 50; 122 static const int kStartupTimeMs = 2000; 123 static const int kMinRunTimeMs = 30000; 124 125 public: 126 VideoRtcpAndSyncObserver(Clock* clock, 127 int voe_channel, 128 VoEVideoSync* voe_sync, 129 SyncRtcpObserver* audio_observer) 130 : SyncRtcpObserver(FakeNetworkPipe::Config()), 131 clock_(clock), 132 voe_channel_(voe_channel), 133 voe_sync_(voe_sync), 134 audio_observer_(audio_observer), 135 creation_time_ms_(clock_->TimeInMilliseconds()), 136 first_time_in_sync_(-1) {} 137 138 virtual void RenderFrame(const I420VideoFrame& video_frame, 139 int time_to_render_ms) OVERRIDE { 140 int64_t now_ms = clock_->TimeInMilliseconds(); 141 uint32_t playout_timestamp = 0; 142 if (voe_sync_->GetPlayoutTimestamp(voe_channel_, playout_timestamp) != 0) 143 return; 144 int64_t latest_audio_ntp = 145 audio_observer_->RtpTimestampToNtp(playout_timestamp); 146 int64_t latest_video_ntp = RtpTimestampToNtp(video_frame.timestamp()); 147 if (latest_audio_ntp < 0 || latest_video_ntp < 0) 148 return; 149 int time_until_render_ms = 150 std::max(0, static_cast<int>(video_frame.render_time_ms() - now_ms)); 151 latest_video_ntp += time_until_render_ms; 152 int64_t stream_offset = latest_audio_ntp - latest_video_ntp; 153 std::stringstream ss; 154 ss << stream_offset; 155 webrtc::test::PrintResult("stream_offset", 156 "", 157 "synchronization", 158 ss.str(), 159 "ms", 160 false); 161 int64_t time_since_creation = now_ms - creation_time_ms_; 162 // During the first couple of seconds audio and video can falsely be 163 // estimated as being synchronized. We don't want to trigger on those. 164 if (time_since_creation < kStartupTimeMs) 165 return; 166 if (std::abs(latest_audio_ntp - latest_video_ntp) < kInSyncThresholdMs) { 167 if (first_time_in_sync_ == -1) { 168 first_time_in_sync_ = now_ms; 169 webrtc::test::PrintResult("sync_convergence_time", 170 "", 171 "synchronization", 172 time_since_creation, 173 "ms", 174 false); 175 } 176 if (time_since_creation > kMinRunTimeMs) 177 observation_complete_->Set(); 178 } 179 } 180 181 private: 182 Clock* const clock_; 183 int voe_channel_; 184 VoEVideoSync* voe_sync_; 185 SyncRtcpObserver* audio_observer_; 186 int64_t creation_time_ms_; 187 int64_t first_time_in_sync_; 188 }; 189 190 void CallPerfTest::TestAudioVideoSync(bool fec) { 191 class AudioPacketReceiver : public PacketReceiver { 192 public: 193 AudioPacketReceiver(int channel, VoENetwork* voe_network) 194 : channel_(channel), 195 voe_network_(voe_network), 196 parser_(RtpHeaderParser::Create()) {} 197 virtual DeliveryStatus DeliverPacket(const uint8_t* packet, 198 size_t length) OVERRIDE { 199 int ret; 200 if (parser_->IsRtcp(packet, static_cast<int>(length))) { 201 ret = voe_network_->ReceivedRTCPPacket( 202 channel_, packet, static_cast<unsigned int>(length)); 203 } else { 204 ret = voe_network_->ReceivedRTPPacket( 205 channel_, packet, static_cast<unsigned int>(length), PacketTime()); 206 } 207 return ret == 0 ? DELIVERY_OK : DELIVERY_PACKET_ERROR; 208 } 209 210 private: 211 int channel_; 212 VoENetwork* voe_network_; 213 scoped_ptr<RtpHeaderParser> parser_; 214 }; 215 216 VoiceEngine* voice_engine = VoiceEngine::Create(); 217 VoEBase* voe_base = VoEBase::GetInterface(voice_engine); 218 VoECodec* voe_codec = VoECodec::GetInterface(voice_engine); 219 VoENetwork* voe_network = VoENetwork::GetInterface(voice_engine); 220 VoEVideoSync* voe_sync = VoEVideoSync::GetInterface(voice_engine); 221 const std::string audio_filename = 222 test::ResourcePath("voice_engine/audio_long16", "pcm"); 223 ASSERT_STRNE("", audio_filename.c_str()); 224 test::FakeAudioDevice fake_audio_device(Clock::GetRealTimeClock(), 225 audio_filename); 226 EXPECT_EQ(0, voe_base->Init(&fake_audio_device, NULL)); 227 int channel = voe_base->CreateChannel(); 228 229 FakeNetworkPipe::Config net_config; 230 net_config.queue_delay_ms = 500; 231 net_config.loss_percent = 5; 232 SyncRtcpObserver audio_observer(net_config); 233 VideoRtcpAndSyncObserver observer(Clock::GetRealTimeClock(), 234 channel, 235 voe_sync, 236 &audio_observer); 237 238 Call::Config receiver_config(observer.ReceiveTransport()); 239 receiver_config.voice_engine = voice_engine; 240 CreateCalls(Call::Config(observer.SendTransport()), receiver_config); 241 242 CodecInst isac = {103, "ISAC", 16000, 480, 1, 32000}; 243 EXPECT_EQ(0, voe_codec->SetSendCodec(channel, isac)); 244 245 AudioPacketReceiver voe_packet_receiver(channel, voe_network); 246 audio_observer.SetReceivers(&voe_packet_receiver, &voe_packet_receiver); 247 248 internal::TransportAdapter transport_adapter(audio_observer.SendTransport()); 249 transport_adapter.Enable(); 250 EXPECT_EQ(0, 251 voe_network->RegisterExternalTransport(channel, transport_adapter)); 252 253 observer.SetReceivers(receiver_call_->Receiver(), sender_call_->Receiver()); 254 255 test::FakeDecoder fake_decoder; 256 257 CreateSendConfig(1); 258 CreateMatchingReceiveConfigs(); 259 260 send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs; 261 if (fec) { 262 send_config_.rtp.fec.red_payload_type = kRedPayloadType; 263 send_config_.rtp.fec.ulpfec_payload_type = kUlpfecPayloadType; 264 receive_configs_[0].rtp.fec.red_payload_type = kRedPayloadType; 265 receive_configs_[0].rtp.fec.ulpfec_payload_type = kUlpfecPayloadType; 266 } 267 receive_configs_[0].rtp.nack.rtp_history_ms = 1000; 268 receive_configs_[0].renderer = &observer; 269 receive_configs_[0].audio_channel_id = channel; 270 271 CreateStreams(); 272 273 CreateFrameGeneratorCapturer(); 274 275 Start(); 276 277 fake_audio_device.Start(); 278 EXPECT_EQ(0, voe_base->StartPlayout(channel)); 279 EXPECT_EQ(0, voe_base->StartReceive(channel)); 280 EXPECT_EQ(0, voe_base->StartSend(channel)); 281 282 EXPECT_EQ(kEventSignaled, observer.Wait()) 283 << "Timed out while waiting for audio and video to be synchronized."; 284 285 EXPECT_EQ(0, voe_base->StopSend(channel)); 286 EXPECT_EQ(0, voe_base->StopReceive(channel)); 287 EXPECT_EQ(0, voe_base->StopPlayout(channel)); 288 fake_audio_device.Stop(); 289 290 Stop(); 291 observer.StopSending(); 292 audio_observer.StopSending(); 293 294 voe_base->DeleteChannel(channel); 295 voe_base->Release(); 296 voe_codec->Release(); 297 voe_network->Release(); 298 voe_sync->Release(); 299 300 DestroyStreams(); 301 302 VoiceEngine::Delete(voice_engine); 303 } 304 305 TEST_F(CallPerfTest, PlaysOutAudioAndVideoInSync) { 306 TestAudioVideoSync(false); 307 } 308 309 TEST_F(CallPerfTest, PlaysOutAudioAndVideoInSyncWithFec) { 310 TestAudioVideoSync(true); 311 } 312 313 void CallPerfTest::TestCaptureNtpTime(const FakeNetworkPipe::Config& net_config, 314 int threshold_ms, 315 int start_time_ms, 316 int run_time_ms) { 317 class CaptureNtpTimeObserver : public test::EndToEndTest, 318 public VideoRenderer { 319 public: 320 CaptureNtpTimeObserver(const FakeNetworkPipe::Config& config, 321 int threshold_ms, 322 int start_time_ms, 323 int run_time_ms) 324 : EndToEndTest(kLongTimeoutMs, config), 325 clock_(Clock::GetRealTimeClock()), 326 threshold_ms_(threshold_ms), 327 start_time_ms_(start_time_ms), 328 run_time_ms_(run_time_ms), 329 creation_time_ms_(clock_->TimeInMilliseconds()), 330 capturer_(NULL), 331 rtp_start_timestamp_set_(false), 332 rtp_start_timestamp_(0) {} 333 334 private: 335 virtual void RenderFrame(const I420VideoFrame& video_frame, 336 int time_to_render_ms) OVERRIDE { 337 if (video_frame.ntp_time_ms() <= 0) { 338 // Haven't got enough RTCP SR in order to calculate the capture ntp 339 // time. 340 return; 341 } 342 343 int64_t now_ms = clock_->TimeInMilliseconds(); 344 int64_t time_since_creation = now_ms - creation_time_ms_; 345 if (time_since_creation < start_time_ms_) { 346 // Wait for |start_time_ms_| before start measuring. 347 return; 348 } 349 350 if (time_since_creation > run_time_ms_) { 351 observation_complete_->Set(); 352 } 353 354 FrameCaptureTimeList::iterator iter = 355 capture_time_list_.find(video_frame.timestamp()); 356 EXPECT_TRUE(iter != capture_time_list_.end()); 357 358 // The real capture time has been wrapped to uint32_t before converted 359 // to rtp timestamp in the sender side. So here we convert the estimated 360 // capture time to a uint32_t 90k timestamp also for comparing. 361 uint32_t estimated_capture_timestamp = 362 90 * static_cast<uint32_t>(video_frame.ntp_time_ms()); 363 uint32_t real_capture_timestamp = iter->second; 364 int time_offset_ms = real_capture_timestamp - estimated_capture_timestamp; 365 time_offset_ms = time_offset_ms / 90; 366 std::stringstream ss; 367 ss << time_offset_ms; 368 369 webrtc::test::PrintResult( 370 "capture_ntp_time", "", "real - estimated", ss.str(), "ms", true); 371 EXPECT_TRUE(std::abs(time_offset_ms) < threshold_ms_); 372 } 373 374 virtual Action OnSendRtp(const uint8_t* packet, size_t length) { 375 RTPHeader header; 376 EXPECT_TRUE(parser_->Parse(packet, length, &header)); 377 378 if (!rtp_start_timestamp_set_) { 379 // Calculate the rtp timestamp offset in order to calculate the real 380 // capture time. 381 uint32_t first_capture_timestamp = 382 90 * static_cast<uint32_t>(capturer_->first_frame_capture_time()); 383 rtp_start_timestamp_ = header.timestamp - first_capture_timestamp; 384 rtp_start_timestamp_set_ = true; 385 } 386 387 uint32_t capture_timestamp = header.timestamp - rtp_start_timestamp_; 388 capture_time_list_.insert( 389 capture_time_list_.end(), 390 std::make_pair(header.timestamp, capture_timestamp)); 391 return SEND_PACKET; 392 } 393 394 virtual void OnFrameGeneratorCapturerCreated( 395 test::FrameGeneratorCapturer* frame_generator_capturer) OVERRIDE { 396 capturer_ = frame_generator_capturer; 397 } 398 399 virtual void ModifyConfigs( 400 VideoSendStream::Config* send_config, 401 std::vector<VideoReceiveStream::Config>* receive_configs, 402 VideoEncoderConfig* encoder_config) OVERRIDE { 403 (*receive_configs)[0].renderer = this; 404 // Enable the receiver side rtt calculation. 405 (*receive_configs)[0].rtp.rtcp_xr.receiver_reference_time_report = true; 406 } 407 408 virtual void PerformTest() OVERRIDE { 409 EXPECT_EQ(kEventSignaled, Wait()) << "Timed out while waiting for " 410 "estimated capture NTP time to be " 411 "within bounds."; 412 } 413 414 Clock* clock_; 415 int threshold_ms_; 416 int start_time_ms_; 417 int run_time_ms_; 418 int64_t creation_time_ms_; 419 test::FrameGeneratorCapturer* capturer_; 420 bool rtp_start_timestamp_set_; 421 uint32_t rtp_start_timestamp_; 422 typedef std::map<uint32_t, uint32_t> FrameCaptureTimeList; 423 FrameCaptureTimeList capture_time_list_; 424 } test(net_config, threshold_ms, start_time_ms, run_time_ms); 425 426 RunBaseTest(&test); 427 } 428 429 TEST_F(CallPerfTest, CaptureNtpTimeWithNetworkDelay) { 430 FakeNetworkPipe::Config net_config; 431 net_config.queue_delay_ms = 100; 432 // TODO(wu): lower the threshold as the calculation/estimatation becomes more 433 // accurate. 434 const int kThresholdMs = 100; 435 const int kStartTimeMs = 10000; 436 const int kRunTimeMs = 20000; 437 TestCaptureNtpTime(net_config, kThresholdMs, kStartTimeMs, kRunTimeMs); 438 } 439 440 TEST_F(CallPerfTest, CaptureNtpTimeWithNetworkJitter) { 441 FakeNetworkPipe::Config net_config; 442 net_config.queue_delay_ms = 100; 443 net_config.delay_standard_deviation_ms = 10; 444 // TODO(wu): lower the threshold as the calculation/estimatation becomes more 445 // accurate. 446 const int kThresholdMs = 100; 447 const int kStartTimeMs = 10000; 448 const int kRunTimeMs = 20000; 449 TestCaptureNtpTime(net_config, kThresholdMs, kStartTimeMs, kRunTimeMs); 450 } 451 452 TEST_F(CallPerfTest, RegisterCpuOveruseObserver) { 453 // Verifies that either a normal or overuse callback is triggered. 454 class OveruseCallbackObserver : public test::SendTest, 455 public webrtc::OveruseCallback { 456 public: 457 OveruseCallbackObserver() : SendTest(kLongTimeoutMs) {} 458 459 virtual void OnOveruse() OVERRIDE { 460 observation_complete_->Set(); 461 } 462 463 virtual void OnNormalUse() OVERRIDE { 464 observation_complete_->Set(); 465 } 466 467 virtual Call::Config GetSenderCallConfig() OVERRIDE { 468 Call::Config config(SendTransport()); 469 config.overuse_callback = this; 470 return config; 471 } 472 473 virtual void PerformTest() OVERRIDE { 474 EXPECT_EQ(kEventSignaled, Wait()) 475 << "Timed out before receiving an overuse callback."; 476 } 477 } test; 478 479 RunBaseTest(&test); 480 } 481 482 void CallPerfTest::TestMinTransmitBitrate(bool pad_to_min_bitrate) { 483 static const int kMaxEncodeBitrateKbps = 30; 484 static const int kMinTransmitBitrateBps = 150000; 485 static const int kMinAcceptableTransmitBitrate = 130; 486 static const int kMaxAcceptableTransmitBitrate = 170; 487 static const int kNumBitrateObservationsInRange = 100; 488 class BitrateObserver : public test::EndToEndTest, public PacketReceiver { 489 public: 490 explicit BitrateObserver(bool using_min_transmit_bitrate) 491 : EndToEndTest(kLongTimeoutMs), 492 send_stream_(NULL), 493 send_transport_receiver_(NULL), 494 pad_to_min_bitrate_(using_min_transmit_bitrate), 495 num_bitrate_observations_in_range_(0) {} 496 497 private: 498 virtual void SetReceivers(PacketReceiver* send_transport_receiver, 499 PacketReceiver* receive_transport_receiver) 500 OVERRIDE { 501 send_transport_receiver_ = send_transport_receiver; 502 test::RtpRtcpObserver::SetReceivers(this, receive_transport_receiver); 503 } 504 505 virtual DeliveryStatus DeliverPacket(const uint8_t* packet, 506 size_t length) OVERRIDE { 507 VideoSendStream::Stats stats = send_stream_->GetStats(); 508 if (stats.substreams.size() > 0) { 509 assert(stats.substreams.size() == 1); 510 int bitrate_kbps = stats.substreams.begin()->second.bitrate_bps / 1000; 511 if (bitrate_kbps > 0) { 512 test::PrintResult( 513 "bitrate_stats_", 514 (pad_to_min_bitrate_ ? "min_transmit_bitrate" 515 : "without_min_transmit_bitrate"), 516 "bitrate_kbps", 517 static_cast<size_t>(bitrate_kbps), 518 "kbps", 519 false); 520 if (pad_to_min_bitrate_) { 521 if (bitrate_kbps > kMinAcceptableTransmitBitrate && 522 bitrate_kbps < kMaxAcceptableTransmitBitrate) { 523 ++num_bitrate_observations_in_range_; 524 } 525 } else { 526 // Expect bitrate stats to roughly match the max encode bitrate. 527 if (bitrate_kbps > kMaxEncodeBitrateKbps - 5 && 528 bitrate_kbps < kMaxEncodeBitrateKbps + 5) { 529 ++num_bitrate_observations_in_range_; 530 } 531 } 532 if (num_bitrate_observations_in_range_ == 533 kNumBitrateObservationsInRange) 534 observation_complete_->Set(); 535 } 536 } 537 return send_transport_receiver_->DeliverPacket(packet, length); 538 } 539 540 virtual void OnStreamsCreated( 541 VideoSendStream* send_stream, 542 const std::vector<VideoReceiveStream*>& receive_streams) OVERRIDE { 543 send_stream_ = send_stream; 544 } 545 546 virtual void ModifyConfigs( 547 VideoSendStream::Config* send_config, 548 std::vector<VideoReceiveStream::Config>* receive_configs, 549 VideoEncoderConfig* encoder_config) OVERRIDE { 550 if (pad_to_min_bitrate_) { 551 send_config->rtp.min_transmit_bitrate_bps = kMinTransmitBitrateBps; 552 } else { 553 assert(send_config->rtp.min_transmit_bitrate_bps == 0); 554 } 555 } 556 557 virtual void PerformTest() OVERRIDE { 558 EXPECT_EQ(kEventSignaled, Wait()) 559 << "Timeout while waiting for send-bitrate stats."; 560 } 561 562 VideoSendStream* send_stream_; 563 PacketReceiver* send_transport_receiver_; 564 const bool pad_to_min_bitrate_; 565 int num_bitrate_observations_in_range_; 566 } test(pad_to_min_bitrate); 567 568 fake_encoder_.SetMaxBitrate(kMaxEncodeBitrateKbps); 569 RunBaseTest(&test); 570 } 571 572 TEST_F(CallPerfTest, PadsToMinTransmitBitrate) { TestMinTransmitBitrate(true); } 573 574 TEST_F(CallPerfTest, NoPadWithoutMinTransmitBitrate) { 575 TestMinTransmitBitrate(false); 576 } 577 578 } // namespace webrtc 579