1 /* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "webrtc/modules/remote_bitrate_estimator/test/bwe_test.h" 12 13 #include <sstream> 14 15 #include "webrtc/base/arraysize.h" 16 #include "webrtc/base/common.h" 17 #include "webrtc/base/scoped_ptr.h" 18 #include "webrtc/modules/include/module_common_types.h" 19 #include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h" 20 #include "webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h" 21 #include "webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h" 22 #include "webrtc/modules/remote_bitrate_estimator/test/packet_sender.h" 23 #include "webrtc/system_wrappers/include/clock.h" 24 #include "webrtc/test/testsupport/perf_test.h" 25 26 using std::string; 27 using std::vector; 28 29 namespace webrtc { 30 namespace testing { 31 namespace bwe { 32 33 PacketProcessorRunner::PacketProcessorRunner(PacketProcessor* processor) 34 : processor_(processor) { 35 } 36 37 PacketProcessorRunner::~PacketProcessorRunner() { 38 for (Packet* packet : queue_) 39 delete packet; 40 } 41 42 bool PacketProcessorRunner::RunsProcessor( 43 const PacketProcessor* processor) const { 44 return processor == processor_; 45 } 46 47 void PacketProcessorRunner::RunFor(int64_t time_ms, 48 int64_t time_now_ms, 49 Packets* in_out) { 50 Packets to_process; 51 FindPacketsToProcess(processor_->flow_ids(), in_out, &to_process); 52 processor_->RunFor(time_ms, &to_process); 53 QueuePackets(&to_process, time_now_ms * 1000); 54 if (!to_process.empty()) { 55 processor_->Plot(to_process.back()->send_time_ms()); 56 } 57 in_out->merge(to_process, DereferencingComparator<Packet>); 58 } 59 60 void PacketProcessorRunner::FindPacketsToProcess(const FlowIds& flow_ids, 61 Packets* in, 62 Packets* out) { 63 assert(out->empty()); 64 for (Packets::iterator it = in->begin(); it != in->end();) { 65 // TODO(holmer): Further optimize this by looking for consecutive flow ids 66 // in the packet list and only doing the binary search + splice once for a 67 // sequence. 68 if (flow_ids.find((*it)->flow_id()) != flow_ids.end()) { 69 Packets::iterator next = it; 70 ++next; 71 out->splice(out->end(), *in, it); 72 it = next; 73 } else { 74 ++it; 75 } 76 } 77 } 78 79 void PacketProcessorRunner::QueuePackets(Packets* batch, 80 int64_t end_of_batch_time_us) { 81 queue_.merge(*batch, DereferencingComparator<Packet>); 82 if (queue_.empty()) { 83 return; 84 } 85 Packets::iterator it = queue_.begin(); 86 for (; it != queue_.end(); ++it) { 87 if ((*it)->send_time_us() > end_of_batch_time_us) { 88 break; 89 } 90 } 91 Packets to_transfer; 92 to_transfer.splice(to_transfer.begin(), queue_, queue_.begin(), it); 93 batch->merge(to_transfer, DereferencingComparator<Packet>); 94 } 95 96 // Plot link capacity by default. 97 BweTest::BweTest() : BweTest(true) { 98 } 99 100 BweTest::BweTest(bool plot_capacity) 101 : run_time_ms_(0), 102 time_now_ms_(-1), 103 simulation_interval_ms_(-1), 104 plot_total_available_capacity_(plot_capacity) { 105 links_.push_back(&uplink_); 106 links_.push_back(&downlink_); 107 } 108 109 BweTest::~BweTest() { 110 for (Packet* packet : packets_) 111 delete packet; 112 } 113 114 void BweTest::SetUp() { 115 const ::testing::TestInfo* const test_info = 116 ::testing::UnitTest::GetInstance()->current_test_info(); 117 string test_name = 118 string(test_info->test_case_name()) + "_" + string(test_info->name()); 119 BWE_TEST_LOGGING_GLOBAL_CONTEXT(test_name); 120 BWE_TEST_LOGGING_GLOBAL_ENABLE(false); 121 } 122 123 void Link::AddPacketProcessor(PacketProcessor* processor, 124 ProcessorType processor_type) { 125 assert(processor); 126 switch (processor_type) { 127 case kSender: 128 senders_.push_back(static_cast<PacketSender*>(processor)); 129 break; 130 case kReceiver: 131 receivers_.push_back(static_cast<PacketReceiver*>(processor)); 132 break; 133 case kRegular: 134 break; 135 } 136 processors_.push_back(PacketProcessorRunner(processor)); 137 } 138 139 void Link::RemovePacketProcessor(PacketProcessor* processor) { 140 for (vector<PacketProcessorRunner>::iterator it = processors_.begin(); 141 it != processors_.end(); ++it) { 142 if (it->RunsProcessor(processor)) { 143 processors_.erase(it); 144 return; 145 } 146 } 147 assert(false); 148 } 149 150 // Ownership of the created packets is handed over to the caller. 151 void Link::Run(int64_t run_for_ms, int64_t now_ms, Packets* packets) { 152 for (auto& processor : processors_) { 153 processor.RunFor(run_for_ms, now_ms, packets); 154 } 155 } 156 157 void BweTest::VerboseLogging(bool enable) { 158 BWE_TEST_LOGGING_GLOBAL_ENABLE(enable); 159 } 160 161 void BweTest::RunFor(int64_t time_ms) { 162 // Set simulation interval from first packet sender. 163 // TODO(holmer): Support different feedback intervals for different flows. 164 if (!uplink_.senders().empty()) { 165 simulation_interval_ms_ = uplink_.senders()[0]->GetFeedbackIntervalMs(); 166 } else if (!downlink_.senders().empty()) { 167 simulation_interval_ms_ = downlink_.senders()[0]->GetFeedbackIntervalMs(); 168 } 169 assert(simulation_interval_ms_ > 0); 170 if (time_now_ms_ == -1) { 171 time_now_ms_ = simulation_interval_ms_; 172 } 173 for (run_time_ms_ += time_ms; 174 time_now_ms_ <= run_time_ms_ - simulation_interval_ms_; 175 time_now_ms_ += simulation_interval_ms_) { 176 // Packets are first generated on the first link, passed through all the 177 // PacketProcessors and PacketReceivers. The PacketReceivers produces 178 // FeedbackPackets which are then processed by the next link, where they 179 // at some point will be consumed by a PacketSender. 180 for (Link* link : links_) 181 link->Run(simulation_interval_ms_, time_now_ms_, &packets_); 182 } 183 } 184 185 string BweTest::GetTestName() const { 186 const ::testing::TestInfo* const test_info = 187 ::testing::UnitTest::GetInstance()->current_test_info(); 188 return string(test_info->name()); 189 } 190 191 void BweTest::PrintResults(double max_throughput_kbps, 192 Stats<double> throughput_kbps, 193 int flow_id, 194 Stats<double> flow_delay_ms, 195 Stats<double> flow_throughput_kbps) { 196 std::map<int, Stats<double>> flow_delays_ms; 197 flow_delays_ms[flow_id] = flow_delay_ms; 198 std::map<int, Stats<double>> flow_throughputs_kbps; 199 flow_throughputs_kbps[flow_id] = flow_throughput_kbps; 200 PrintResults(max_throughput_kbps, throughput_kbps, flow_delays_ms, 201 flow_throughputs_kbps); 202 } 203 204 void BweTest::PrintResults(double max_throughput_kbps, 205 Stats<double> throughput_kbps, 206 std::map<int, Stats<double>> flow_delay_ms, 207 std::map<int, Stats<double>> flow_throughput_kbps) { 208 double utilization = throughput_kbps.GetMean() / max_throughput_kbps; 209 webrtc::test::PrintResult("BwePerformance", GetTestName(), "Utilization", 210 utilization * 100.0, "%", false); 211 std::stringstream ss; 212 ss << throughput_kbps.GetStdDev() / throughput_kbps.GetMean(); 213 webrtc::test::PrintResult("BwePerformance", GetTestName(), 214 "Utilization var coeff", ss.str(), "", false); 215 for (auto& kv : flow_throughput_kbps) { 216 ss.str(""); 217 ss << "Throughput flow " << kv.first; 218 webrtc::test::PrintResultMeanAndError("BwePerformance", GetTestName(), 219 ss.str(), kv.second.AsString(), 220 "kbps", false); 221 } 222 for (auto& kv : flow_delay_ms) { 223 ss.str(""); 224 ss << "Delay flow " << kv.first; 225 webrtc::test::PrintResultMeanAndError("BwePerformance", GetTestName(), 226 ss.str(), kv.second.AsString(), "ms", 227 false); 228 } 229 double fairness_index = 1.0; 230 if (!flow_throughput_kbps.empty()) { 231 double squared_bitrate_sum = 0.0; 232 fairness_index = 0.0; 233 for (auto kv : flow_throughput_kbps) { 234 squared_bitrate_sum += kv.second.GetMean() * kv.second.GetMean(); 235 fairness_index += kv.second.GetMean(); 236 } 237 fairness_index *= fairness_index; 238 fairness_index /= flow_throughput_kbps.size() * squared_bitrate_sum; 239 } 240 webrtc::test::PrintResult("BwePerformance", GetTestName(), "Fairness", 241 fairness_index * 100, "%", false); 242 } 243 244 void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type, 245 size_t num_media_flows, 246 size_t num_tcp_flows, 247 int64_t run_time_seconds, 248 uint32_t capacity_kbps, 249 int64_t max_delay_ms, 250 int64_t rtt_ms, 251 int64_t max_jitter_ms, 252 const int64_t* offsets_ms) { 253 RunFairnessTest(bwe_type, num_media_flows, num_tcp_flows, run_time_seconds, 254 capacity_kbps, max_delay_ms, rtt_ms, max_jitter_ms, 255 offsets_ms, "Fairness_test", bwe_names[bwe_type]); 256 } 257 258 void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type, 259 size_t num_media_flows, 260 size_t num_tcp_flows, 261 int64_t run_time_seconds, 262 uint32_t capacity_kbps, 263 int64_t max_delay_ms, 264 int64_t rtt_ms, 265 int64_t max_jitter_ms, 266 const int64_t* offsets_ms, 267 const std::string& title, 268 const std::string& flow_name) { 269 std::set<int> all_flow_ids; 270 std::set<int> media_flow_ids; 271 std::set<int> tcp_flow_ids; 272 int next_flow_id = 0; 273 for (size_t i = 0; i < num_media_flows; ++i) { 274 media_flow_ids.insert(next_flow_id); 275 all_flow_ids.insert(next_flow_id); 276 ++next_flow_id; 277 } 278 for (size_t i = 0; i < num_tcp_flows; ++i) { 279 tcp_flow_ids.insert(next_flow_id); 280 all_flow_ids.insert(next_flow_id); 281 ++next_flow_id; 282 } 283 284 std::vector<VideoSource*> sources; 285 std::vector<PacketSender*> senders; 286 std::vector<MetricRecorder*> metric_recorders; 287 288 int64_t max_offset_ms = 0; 289 290 for (int media_flow : media_flow_ids) { 291 sources.push_back(new AdaptiveVideoSource(media_flow, 30, 300, 0, 292 offsets_ms[media_flow])); 293 senders.push_back(new PacedVideoSender(&uplink_, sources.back(), bwe_type)); 294 max_offset_ms = std::max(max_offset_ms, offsets_ms[media_flow]); 295 } 296 297 for (int tcp_flow : tcp_flow_ids) { 298 senders.push_back(new TcpSender(&uplink_, tcp_flow, offsets_ms[tcp_flow])); 299 max_offset_ms = std::max(max_offset_ms, offsets_ms[tcp_flow]); 300 } 301 302 ChokeFilter choke(&uplink_, all_flow_ids); 303 choke.set_capacity_kbps(capacity_kbps); 304 choke.set_max_delay_ms(max_delay_ms); 305 LinkShare link_share(&choke); 306 307 int64_t one_way_delay_ms = rtt_ms / 2; 308 DelayFilter delay_uplink(&uplink_, all_flow_ids); 309 delay_uplink.SetOneWayDelayMs(one_way_delay_ms); 310 311 JitterFilter jitter(&uplink_, all_flow_ids); 312 jitter.SetMaxJitter(max_jitter_ms); 313 314 std::vector<RateCounterFilter*> rate_counters; 315 for (int flow : media_flow_ids) { 316 rate_counters.push_back( 317 new RateCounterFilter(&uplink_, flow, "Receiver", bwe_names[bwe_type])); 318 } 319 for (int flow : tcp_flow_ids) { 320 rate_counters.push_back(new RateCounterFilter(&uplink_, flow, "Receiver", 321 bwe_names[kTcpEstimator])); 322 } 323 324 RateCounterFilter total_utilization( 325 &uplink_, all_flow_ids, "total_utilization", "Total_link_utilization"); 326 327 std::vector<PacketReceiver*> receivers; 328 // Delays is being plotted only for the first flow. 329 // To plot all of them, replace "i == 0" with "true" on new PacketReceiver(). 330 for (int media_flow : media_flow_ids) { 331 metric_recorders.push_back( 332 new MetricRecorder(bwe_names[bwe_type], static_cast<int>(media_flow), 333 senders[media_flow], &link_share)); 334 receivers.push_back(new PacketReceiver(&uplink_, media_flow, bwe_type, 335 media_flow == 0, false, 336 metric_recorders[media_flow])); 337 metric_recorders[media_flow]->set_plot_available_capacity( 338 media_flow == 0 && plot_total_available_capacity_); 339 metric_recorders[media_flow]->set_start_computing_metrics_ms(max_offset_ms); 340 } 341 // Delays is not being plotted only for TCP flows. To plot all of them, 342 // replace first "false" occurence with "true" on new PacketReceiver(). 343 for (int tcp_flow : tcp_flow_ids) { 344 metric_recorders.push_back( 345 new MetricRecorder(bwe_names[kTcpEstimator], static_cast<int>(tcp_flow), 346 senders[tcp_flow], &link_share)); 347 receivers.push_back(new PacketReceiver(&uplink_, tcp_flow, kTcpEstimator, 348 false, false, 349 metric_recorders[tcp_flow])); 350 metric_recorders[tcp_flow]->set_plot_available_capacity( 351 tcp_flow == 0 && plot_total_available_capacity_); 352 } 353 354 DelayFilter delay_downlink(&downlink_, all_flow_ids); 355 delay_downlink.SetOneWayDelayMs(one_way_delay_ms); 356 357 RunFor(run_time_seconds * 1000); 358 359 std::map<int, Stats<double>> flow_throughput_kbps; 360 for (RateCounterFilter* rate_counter : rate_counters) { 361 int flow_id = *rate_counter->flow_ids().begin(); 362 flow_throughput_kbps[flow_id] = rate_counter->GetBitrateStats(); 363 } 364 365 std::map<int, Stats<double>> flow_delay_ms; 366 for (PacketReceiver* receiver : receivers) { 367 int flow_id = *receiver->flow_ids().begin(); 368 flow_delay_ms[flow_id] = receiver->GetDelayStats(); 369 } 370 371 PrintResults(capacity_kbps, total_utilization.GetBitrateStats(), 372 flow_delay_ms, flow_throughput_kbps); 373 374 for (int i : all_flow_ids) { 375 metric_recorders[i]->PlotThroughputHistogram( 376 title, flow_name, static_cast<int>(num_media_flows), 0); 377 378 metric_recorders[i]->PlotLossHistogram(title, flow_name, 379 static_cast<int>(num_media_flows), 380 receivers[i]->GlobalPacketLoss()); 381 } 382 383 // Pointless to show delay histogram for TCP flow. 384 for (int i : media_flow_ids) { 385 metric_recorders[i]->PlotDelayHistogram(title, bwe_names[bwe_type], 386 static_cast<int>(num_media_flows), 387 one_way_delay_ms); 388 BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], one_way_delay_ms, i); 389 } 390 391 for (VideoSource* source : sources) 392 delete source; 393 for (PacketSender* sender : senders) 394 delete sender; 395 for (RateCounterFilter* rate_counter : rate_counters) 396 delete rate_counter; 397 for (PacketReceiver* receiver : receivers) 398 delete receiver; 399 for (MetricRecorder* recorder : metric_recorders) 400 delete recorder; 401 } 402 403 void BweTest::RunChoke(BandwidthEstimatorType bwe_type, 404 std::vector<int> capacities_kbps) { 405 int flow_id = bwe_type; 406 AdaptiveVideoSource source(flow_id, 30, 300, 0, 0); 407 VideoSender sender(&uplink_, &source, bwe_type); 408 ChokeFilter choke(&uplink_, flow_id); 409 LinkShare link_share(&choke); 410 MetricRecorder metric_recorder(bwe_names[bwe_type], flow_id, &sender, 411 &link_share); 412 PacketReceiver receiver(&uplink_, flow_id, bwe_type, true, false, 413 &metric_recorder); 414 metric_recorder.set_plot_available_capacity(plot_total_available_capacity_); 415 416 choke.set_max_delay_ms(500); 417 const int64_t kRunTimeMs = 60 * 1000; 418 419 std::stringstream title("Choke"); 420 char delimiter = '_'; 421 422 for (auto it = capacities_kbps.begin(); it != capacities_kbps.end(); ++it) { 423 choke.set_capacity_kbps(*it); 424 RunFor(kRunTimeMs); 425 title << delimiter << (*it); 426 delimiter = '-'; 427 } 428 429 title << "_kbps,_" << (kRunTimeMs / 1000) << "s_each"; 430 metric_recorder.PlotThroughputHistogram(title.str(), bwe_names[bwe_type], 1, 431 0); 432 metric_recorder.PlotDelayHistogram(title.str(), bwe_names[bwe_type], 1, 0); 433 // receiver.PlotLossHistogram(title, bwe_names[bwe_type], 1); 434 // receiver.PlotObjectiveHistogram(title, bwe_names[bwe_type], 1); 435 } 436 437 // 5.1. Single Video and Audio media traffic, forward direction. 438 void BweTest::RunVariableCapacity1SingleFlow(BandwidthEstimatorType bwe_type) { 439 const int kFlowId = 0; // Arbitrary value. 440 AdaptiveVideoSource source(kFlowId, 30, 300, 0, 0); 441 PacedVideoSender sender(&uplink_, &source, bwe_type); 442 443 DefaultEvaluationFilter up_filter(&uplink_, kFlowId); 444 LinkShare link_share(&(up_filter.choke)); 445 MetricRecorder metric_recorder(bwe_names[bwe_type], kFlowId, &sender, 446 &link_share); 447 448 PacketReceiver receiver(&uplink_, kFlowId, bwe_type, true, true, 449 &metric_recorder); 450 451 metric_recorder.set_plot_available_capacity(plot_total_available_capacity_); 452 453 DelayFilter down_filter(&downlink_, kFlowId); 454 down_filter.SetOneWayDelayMs(kOneWayDelayMs); 455 456 // Test also with one way propagation delay = 100ms. 457 // up_filter.delay.SetOneWayDelayMs(100); 458 // down_filter.SetOneWayDelayMs(100); 459 460 up_filter.choke.set_capacity_kbps(1000); 461 RunFor(40 * 1000); // 0-40s. 462 up_filter.choke.set_capacity_kbps(2500); 463 RunFor(20 * 1000); // 40-60s. 464 up_filter.choke.set_capacity_kbps(600); 465 RunFor(20 * 1000); // 60-80s. 466 up_filter.choke.set_capacity_kbps(1000); 467 RunFor(20 * 1000); // 80-100s. 468 469 std::string title("5.1_Variable_capacity_single_flow"); 470 metric_recorder.PlotThroughputHistogram(title, bwe_names[bwe_type], 1, 0); 471 metric_recorder.PlotDelayHistogram(title, bwe_names[bwe_type], 1, 472 kOneWayDelayMs); 473 metric_recorder.PlotLossHistogram(title, bwe_names[bwe_type], 1, 474 receiver.GlobalPacketLoss()); 475 BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, kFlowId); 476 } 477 478 // 5.2. Two forward direction competing flows, variable capacity. 479 void BweTest::RunVariableCapacity2MultipleFlows(BandwidthEstimatorType bwe_type, 480 size_t num_flows) { 481 std::vector<VideoSource*> sources; 482 std::vector<PacketSender*> senders; 483 std::vector<MetricRecorder*> metric_recorders; 484 std::vector<PacketReceiver*> receivers; 485 486 const int64_t kStartingApartMs = 0; // Flows initialized simultaneously. 487 488 for (size_t i = 0; i < num_flows; ++i) { 489 sources.push_back(new AdaptiveVideoSource(static_cast<int>(i), 30, 300, 0, 490 i * kStartingApartMs)); 491 senders.push_back(new VideoSender(&uplink_, sources[i], bwe_type)); 492 } 493 494 FlowIds flow_ids = CreateFlowIdRange(0, static_cast<int>(num_flows - 1)); 495 496 DefaultEvaluationFilter up_filter(&uplink_, flow_ids); 497 LinkShare link_share(&(up_filter.choke)); 498 499 RateCounterFilter total_utilization(&uplink_, flow_ids, "Total_utilization", 500 "Total_link_utilization"); 501 502 // Delays is being plotted only for the first flow. 503 // To plot all of them, replace "i == 0" with "true" on new PacketReceiver(). 504 for (size_t i = 0; i < num_flows; ++i) { 505 metric_recorders.push_back(new MetricRecorder( 506 bwe_names[bwe_type], static_cast<int>(i), senders[i], &link_share)); 507 508 receivers.push_back(new PacketReceiver(&uplink_, static_cast<int>(i), 509 bwe_type, i == 0, false, 510 metric_recorders[i])); 511 metric_recorders[i]->set_plot_available_capacity( 512 i == 0 && plot_total_available_capacity_); 513 } 514 515 DelayFilter down_filter(&downlink_, flow_ids); 516 down_filter.SetOneWayDelayMs(kOneWayDelayMs); 517 // Test also with one way propagation delay = 100ms. 518 // up_filter.delay.SetOneWayDelayMs(100); 519 // down_filter.SetOneWayDelayMs(100); 520 521 up_filter.choke.set_capacity_kbps(4000); 522 RunFor(25 * 1000); // 0-25s. 523 up_filter.choke.set_capacity_kbps(2000); 524 RunFor(25 * 1000); // 25-50s. 525 up_filter.choke.set_capacity_kbps(3500); 526 RunFor(25 * 1000); // 50-75s. 527 up_filter.choke.set_capacity_kbps(1000); 528 RunFor(25 * 1000); // 75-100s. 529 up_filter.choke.set_capacity_kbps(2000); 530 RunFor(25 * 1000); // 100-125s. 531 532 std::string title("5.2_Variable_capacity_two_flows"); 533 for (size_t i = 0; i < num_flows; ++i) { 534 metric_recorders[i]->PlotThroughputHistogram(title, bwe_names[bwe_type], 535 num_flows, 0); 536 metric_recorders[i]->PlotDelayHistogram(title, bwe_names[bwe_type], 537 num_flows, kOneWayDelayMs); 538 metric_recorders[i]->PlotLossHistogram(title, bwe_names[bwe_type], 539 num_flows, 540 receivers[i]->GlobalPacketLoss()); 541 BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, i); 542 } 543 544 for (VideoSource* source : sources) 545 delete source; 546 for (PacketSender* sender : senders) 547 delete sender; 548 for (MetricRecorder* recorder : metric_recorders) 549 delete recorder; 550 for (PacketReceiver* receiver : receivers) 551 delete receiver; 552 } 553 554 // 5.3. Bi-directional RMCAT flows. 555 void BweTest::RunBidirectionalFlow(BandwidthEstimatorType bwe_type) { 556 enum direction { kForward = 0, kBackward }; 557 const size_t kNumFlows = 2; 558 rtc::scoped_ptr<AdaptiveVideoSource> sources[kNumFlows]; 559 rtc::scoped_ptr<VideoSender> senders[kNumFlows]; 560 rtc::scoped_ptr<MetricRecorder> metric_recorders[kNumFlows]; 561 rtc::scoped_ptr<PacketReceiver> receivers[kNumFlows]; 562 563 sources[kForward].reset(new AdaptiveVideoSource(kForward, 30, 300, 0, 0)); 564 senders[kForward].reset( 565 new VideoSender(&uplink_, sources[kForward].get(), bwe_type)); 566 567 sources[kBackward].reset(new AdaptiveVideoSource(kBackward, 30, 300, 0, 0)); 568 senders[kBackward].reset( 569 new VideoSender(&downlink_, sources[kBackward].get(), bwe_type)); 570 571 DefaultEvaluationFilter up_filter(&uplink_, kForward); 572 LinkShare up_link_share(&(up_filter.choke)); 573 574 metric_recorders[kForward].reset(new MetricRecorder( 575 bwe_names[bwe_type], kForward, senders[kForward].get(), &up_link_share)); 576 receivers[kForward].reset( 577 new PacketReceiver(&uplink_, kForward, bwe_type, true, false, 578 metric_recorders[kForward].get())); 579 580 metric_recorders[kForward].get()->set_plot_available_capacity( 581 plot_total_available_capacity_); 582 583 DefaultEvaluationFilter down_filter(&downlink_, kBackward); 584 LinkShare down_link_share(&(down_filter.choke)); 585 586 metric_recorders[kBackward].reset( 587 new MetricRecorder(bwe_names[bwe_type], kBackward, 588 senders[kBackward].get(), &down_link_share)); 589 receivers[kBackward].reset( 590 new PacketReceiver(&downlink_, kBackward, bwe_type, true, false, 591 metric_recorders[kBackward].get())); 592 593 metric_recorders[kBackward].get()->set_plot_available_capacity( 594 plot_total_available_capacity_); 595 596 // Test also with one way propagation delay = 100ms. 597 // up_filter.delay.SetOneWayDelayMs(100); 598 // down_filter.delay.SetOneWayDelayMs(100); 599 600 up_filter.choke.set_capacity_kbps(2000); 601 down_filter.choke.set_capacity_kbps(2000); 602 RunFor(20 * 1000); // 0-20s. 603 604 up_filter.choke.set_capacity_kbps(1000); 605 RunFor(15 * 1000); // 20-35s. 606 607 down_filter.choke.set_capacity_kbps(800); 608 RunFor(5 * 1000); // 35-40s. 609 610 up_filter.choke.set_capacity_kbps(500); 611 RunFor(20 * 1000); // 40-60s. 612 613 up_filter.choke.set_capacity_kbps(2000); 614 RunFor(10 * 1000); // 60-70s. 615 616 down_filter.choke.set_capacity_kbps(2000); 617 RunFor(30 * 1000); // 70-100s. 618 619 std::string title("5.3_Bidirectional_flows"); 620 for (size_t i = 0; i < kNumFlows; ++i) { 621 metric_recorders[i].get()->PlotThroughputHistogram( 622 title, bwe_names[bwe_type], kNumFlows, 0); 623 metric_recorders[i].get()->PlotDelayHistogram(title, bwe_names[bwe_type], 624 kNumFlows, kOneWayDelayMs); 625 metric_recorders[i].get()->PlotLossHistogram( 626 title, bwe_names[bwe_type], kNumFlows, 627 receivers[i].get()->GlobalPacketLoss()); 628 BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, i); 629 } 630 } 631 632 // 5.4. Three forward direction competing flows, constant capacity. 633 void BweTest::RunSelfFairness(BandwidthEstimatorType bwe_type) { 634 const int kNumRmcatFlows = 3; 635 const int kNumTcpFlows = 0; 636 const int64_t kRunTimeS = 120; 637 const int kLinkCapacity = 3500; 638 639 int64_t max_delay_ms = kMaxQueueingDelayMs; 640 int64_t rtt_ms = 2 * kOneWayDelayMs; 641 642 const int64_t kStartingApartMs = 20 * 1000; 643 int64_t offsets_ms[kNumRmcatFlows]; 644 for (int i = 0; i < kNumRmcatFlows; ++i) { 645 offsets_ms[i] = kStartingApartMs * i; 646 } 647 648 // Test also with one way propagation delay = 100ms. 649 // rtt_ms = 2 * 100; 650 // Test also with bottleneck queue size = 20ms and 1000ms. 651 // max_delay_ms = 20; 652 // max_delay_ms = 1000; 653 654 std::string title("5.4_Self_fairness_test"); 655 656 // Test also with one way propagation delay = 100ms. 657 RunFairnessTest(bwe_type, kNumRmcatFlows, kNumTcpFlows, kRunTimeS, 658 kLinkCapacity, max_delay_ms, rtt_ms, kMaxJitterMs, offsets_ms, 659 title, bwe_names[bwe_type]); 660 } 661 662 // 5.5. Five competing RMCAT flows under different RTTs. 663 void BweTest::RunRoundTripTimeFairness(BandwidthEstimatorType bwe_type) { 664 const int kAllFlowIds[] = {0, 1, 2, 3, 4}; // Five RMCAT flows. 665 const int64_t kAllOneWayDelayMs[] = {10, 25, 50, 100, 150}; 666 const size_t kNumFlows = arraysize(kAllFlowIds); 667 rtc::scoped_ptr<AdaptiveVideoSource> sources[kNumFlows]; 668 rtc::scoped_ptr<VideoSender> senders[kNumFlows]; 669 rtc::scoped_ptr<MetricRecorder> metric_recorders[kNumFlows]; 670 671 // Flows initialized 10 seconds apart. 672 const int64_t kStartingApartMs = 10 * 1000; 673 674 for (size_t i = 0; i < kNumFlows; ++i) { 675 sources[i].reset(new AdaptiveVideoSource(kAllFlowIds[i], 30, 300, 0, 676 i * kStartingApartMs)); 677 senders[i].reset(new VideoSender(&uplink_, sources[i].get(), bwe_type)); 678 } 679 680 ChokeFilter choke_filter(&uplink_, CreateFlowIds(kAllFlowIds, kNumFlows)); 681 LinkShare link_share(&choke_filter); 682 683 JitterFilter jitter_filter(&uplink_, CreateFlowIds(kAllFlowIds, kNumFlows)); 684 685 rtc::scoped_ptr<DelayFilter> up_delay_filters[kNumFlows]; 686 for (size_t i = 0; i < kNumFlows; ++i) { 687 up_delay_filters[i].reset(new DelayFilter(&uplink_, kAllFlowIds[i])); 688 } 689 690 RateCounterFilter total_utilization( 691 &uplink_, CreateFlowIds(kAllFlowIds, kNumFlows), "Total_utilization", 692 "Total_link_utilization"); 693 694 // Delays is being plotted only for the first flow. 695 // To plot all of them, replace "i == 0" with "true" on new PacketReceiver(). 696 rtc::scoped_ptr<PacketReceiver> receivers[kNumFlows]; 697 for (size_t i = 0; i < kNumFlows; ++i) { 698 metric_recorders[i].reset( 699 new MetricRecorder(bwe_names[bwe_type], static_cast<int>(i), 700 senders[i].get(), &link_share)); 701 702 receivers[i].reset(new PacketReceiver(&uplink_, kAllFlowIds[i], bwe_type, 703 i == 0, false, 704 metric_recorders[i].get())); 705 metric_recorders[i].get()->set_start_computing_metrics_ms(kStartingApartMs * 706 (kNumFlows - 1)); 707 metric_recorders[i].get()->set_plot_available_capacity( 708 i == 0 && plot_total_available_capacity_); 709 } 710 711 rtc::scoped_ptr<DelayFilter> down_delay_filters[kNumFlows]; 712 for (size_t i = 0; i < kNumFlows; ++i) { 713 down_delay_filters[i].reset(new DelayFilter(&downlink_, kAllFlowIds[i])); 714 } 715 716 jitter_filter.SetMaxJitter(kMaxJitterMs); 717 choke_filter.set_max_delay_ms(kMaxQueueingDelayMs); 718 719 for (size_t i = 0; i < kNumFlows; ++i) { 720 up_delay_filters[i]->SetOneWayDelayMs(kAllOneWayDelayMs[i]); 721 down_delay_filters[i]->SetOneWayDelayMs(kAllOneWayDelayMs[i]); 722 } 723 724 choke_filter.set_capacity_kbps(3500); 725 726 RunFor(300 * 1000); // 0-300s. 727 728 std::string title("5.5_Round_Trip_Time_Fairness"); 729 for (size_t i = 0; i < kNumFlows; ++i) { 730 metric_recorders[i].get()->PlotThroughputHistogram( 731 title, bwe_names[bwe_type], kNumFlows, 0); 732 metric_recorders[i].get()->PlotDelayHistogram(title, bwe_names[bwe_type], 733 kNumFlows, kOneWayDelayMs); 734 metric_recorders[i].get()->PlotLossHistogram( 735 title, bwe_names[bwe_type], kNumFlows, 736 receivers[i].get()->GlobalPacketLoss()); 737 BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kAllOneWayDelayMs[i], 738 i); 739 } 740 } 741 742 // 5.6. RMCAT Flow competing with a long TCP Flow. 743 void BweTest::RunLongTcpFairness(BandwidthEstimatorType bwe_type) { 744 const size_t kNumRmcatFlows = 1; 745 const size_t kNumTcpFlows = 1; 746 const int64_t kRunTimeS = 120; 747 const int kCapacityKbps = 2000; 748 // Tcp starts at t = 0, media flow at t = 5s. 749 const int64_t kOffSetsMs[] = {5000, 0}; 750 751 int64_t max_delay_ms = kMaxQueueingDelayMs; 752 int64_t rtt_ms = 2 * kOneWayDelayMs; 753 754 // Test also with one way propagation delay = 100ms. 755 // rtt_ms = 2 * 100; 756 // Test also with bottleneck queue size = 20ms and 1000ms. 757 // max_delay_ms = 20; 758 // max_delay_ms = 1000; 759 760 std::string title("5.6_Long_TCP_Fairness"); 761 std::string flow_name(bwe_names[bwe_type] + 'x' + bwe_names[kTcpEstimator]); 762 763 RunFairnessTest(bwe_type, kNumRmcatFlows, kNumTcpFlows, kRunTimeS, 764 kCapacityKbps, max_delay_ms, rtt_ms, kMaxJitterMs, kOffSetsMs, 765 title, flow_name); 766 } 767 768 // 5.7. RMCAT Flows competing with multiple short TCP Flows. 769 void BweTest::RunMultipleShortTcpFairness( 770 BandwidthEstimatorType bwe_type, 771 std::vector<int> tcp_file_sizes_bytes, 772 std::vector<int64_t> tcp_starting_times_ms) { 773 // Two RMCAT flows and ten TCP flows. 774 const int kAllRmcatFlowIds[] = {0, 1}; 775 const int kAllTcpFlowIds[] = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; 776 777 assert(tcp_starting_times_ms.size() == tcp_file_sizes_bytes.size() && 778 tcp_starting_times_ms.size() == arraysize(kAllTcpFlowIds)); 779 780 const size_t kNumRmcatFlows = arraysize(kAllRmcatFlowIds); 781 const size_t kNumTotalFlows = kNumRmcatFlows + arraysize(kAllTcpFlowIds); 782 783 rtc::scoped_ptr<AdaptiveVideoSource> sources[kNumRmcatFlows]; 784 rtc::scoped_ptr<PacketSender> senders[kNumTotalFlows]; 785 rtc::scoped_ptr<MetricRecorder> metric_recorders[kNumTotalFlows]; 786 rtc::scoped_ptr<PacketReceiver> receivers[kNumTotalFlows]; 787 788 // RMCAT Flows are initialized simultaneosly at t=5 seconds. 789 const int64_t kRmcatStartingTimeMs = 5 * 1000; 790 for (size_t id : kAllRmcatFlowIds) { 791 sources[id].reset(new AdaptiveVideoSource(static_cast<int>(id), 30, 300, 0, 792 kRmcatStartingTimeMs)); 793 senders[id].reset(new VideoSender(&uplink_, sources[id].get(), bwe_type)); 794 } 795 796 for (size_t id : kAllTcpFlowIds) { 797 senders[id].reset(new TcpSender(&uplink_, static_cast<int>(id), 798 tcp_starting_times_ms[id - kNumRmcatFlows], 799 tcp_file_sizes_bytes[id - kNumRmcatFlows])); 800 } 801 802 FlowIds flow_ids = CreateFlowIdRange(0, static_cast<int>(kNumTotalFlows - 1)); 803 DefaultEvaluationFilter up_filter(&uplink_, flow_ids); 804 805 LinkShare link_share(&(up_filter.choke)); 806 807 RateCounterFilter total_utilization(&uplink_, flow_ids, "Total_utilization", 808 "Total_link_utilization"); 809 810 // Delays is being plotted only for the first flow. 811 // To plot all of them, replace "i == 0" with "true" on new PacketReceiver(). 812 for (size_t id : kAllRmcatFlowIds) { 813 metric_recorders[id].reset( 814 new MetricRecorder(bwe_names[bwe_type], static_cast<int>(id), 815 senders[id].get(), &link_share)); 816 receivers[id].reset(new PacketReceiver(&uplink_, static_cast<int>(id), 817 bwe_type, id == 0, false, 818 metric_recorders[id].get())); 819 metric_recorders[id].get()->set_start_computing_metrics_ms( 820 kRmcatStartingTimeMs); 821 metric_recorders[id].get()->set_plot_available_capacity( 822 id == 0 && plot_total_available_capacity_); 823 } 824 825 // Delays is not being plotted only for TCP flows. To plot all of them, 826 // replace first "false" occurence with "true" on new PacketReceiver(). 827 for (size_t id : kAllTcpFlowIds) { 828 metric_recorders[id].reset( 829 new MetricRecorder(bwe_names[kTcpEstimator], static_cast<int>(id), 830 senders[id].get(), &link_share)); 831 receivers[id].reset(new PacketReceiver(&uplink_, static_cast<int>(id), 832 kTcpEstimator, false, false, 833 metric_recorders[id].get())); 834 metric_recorders[id].get()->set_plot_available_capacity( 835 id == 0 && plot_total_available_capacity_); 836 } 837 838 DelayFilter down_filter(&downlink_, flow_ids); 839 down_filter.SetOneWayDelayMs(kOneWayDelayMs); 840 841 // Test also with one way propagation delay = 100ms. 842 // up_filter.delay.SetOneWayDelayMs(100); 843 // down_filter.SetOneWayDelayms(100); 844 845 // Test also with bottleneck queue size = 20ms and 1000ms. 846 // up_filter.choke.set_max_delay_ms(20); 847 // up_filter.choke.set_max_delay_ms(1000); 848 849 // Test also with no Jitter: 850 // up_filter.jitter.SetMaxJitter(0); 851 852 up_filter.choke.set_capacity_kbps(2000); 853 854 RunFor(300 * 1000); // 0-300s. 855 856 std::string title("5.7_Multiple_short_TCP_flows"); 857 for (size_t id : kAllRmcatFlowIds) { 858 metric_recorders[id].get()->PlotThroughputHistogram( 859 title, bwe_names[bwe_type], kNumRmcatFlows, 0); 860 metric_recorders[id].get()->PlotDelayHistogram( 861 title, bwe_names[bwe_type], kNumRmcatFlows, kOneWayDelayMs); 862 metric_recorders[id].get()->PlotLossHistogram( 863 title, bwe_names[bwe_type], kNumRmcatFlows, 864 receivers[id].get()->GlobalPacketLoss()); 865 BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, id); 866 } 867 } 868 869 // 5.8. Three forward direction competing flows, constant capacity. 870 // During the test, one of the flows is paused and later resumed. 871 void BweTest::RunPauseResumeFlows(BandwidthEstimatorType bwe_type) { 872 const int kAllFlowIds[] = {0, 1, 2}; // Three RMCAT flows. 873 const size_t kNumFlows = arraysize(kAllFlowIds); 874 875 rtc::scoped_ptr<AdaptiveVideoSource> sources[kNumFlows]; 876 rtc::scoped_ptr<VideoSender> senders[kNumFlows]; 877 rtc::scoped_ptr<MetricRecorder> metric_recorders[kNumFlows]; 878 rtc::scoped_ptr<PacketReceiver> receivers[kNumFlows]; 879 880 // Flows initialized simultaneously. 881 const int64_t kStartingApartMs = 0; 882 883 for (size_t i = 0; i < kNumFlows; ++i) { 884 sources[i].reset(new AdaptiveVideoSource(kAllFlowIds[i], 30, 300, 0, 885 i * kStartingApartMs)); 886 senders[i].reset(new VideoSender(&uplink_, sources[i].get(), bwe_type)); 887 } 888 889 DefaultEvaluationFilter filter(&uplink_, 890 CreateFlowIds(kAllFlowIds, kNumFlows)); 891 892 LinkShare link_share(&(filter.choke)); 893 894 RateCounterFilter total_utilization( 895 &uplink_, CreateFlowIds(kAllFlowIds, kNumFlows), "Total_utilization", 896 "Total_link_utilization"); 897 898 // Delays is being plotted only for the first flow. 899 // To plot all of them, replace "i == 0" with "true" on new PacketReceiver(). 900 for (size_t i = 0; i < kNumFlows; ++i) { 901 metric_recorders[i].reset( 902 new MetricRecorder(bwe_names[bwe_type], static_cast<int>(i), 903 senders[i].get(), &link_share)); 904 receivers[i].reset(new PacketReceiver(&uplink_, kAllFlowIds[i], bwe_type, 905 i == 0, false, 906 metric_recorders[i].get())); 907 metric_recorders[i].get()->set_start_computing_metrics_ms(kStartingApartMs * 908 (kNumFlows - 1)); 909 metric_recorders[i].get()->set_plot_available_capacity( 910 i == 0 && plot_total_available_capacity_); 911 } 912 913 // Test also with one way propagation delay = 100ms. 914 // filter.delay.SetOneWayDelayMs(100); 915 filter.choke.set_capacity_kbps(3500); 916 917 RunFor(40 * 1000); // 0-40s. 918 senders[0].get()->Pause(); 919 RunFor(20 * 1000); // 40-60s. 920 senders[0].get()->Resume(20 * 1000); 921 RunFor(60 * 1000); // 60-120s. 922 923 int64_t paused[] = {20 * 1000, 0, 0}; 924 925 // First flow is being paused, hence having a different optimum. 926 const std::string optima_lines[] = {"1", "2", "2"}; 927 928 std::string title("5.8_Pause_and_resume_media_flow"); 929 for (size_t i = 0; i < kNumFlows; ++i) { 930 metric_recorders[i].get()->PlotThroughputHistogram( 931 title, bwe_names[bwe_type], kNumFlows, paused[i], optima_lines[i]); 932 metric_recorders[i].get()->PlotDelayHistogram(title, bwe_names[bwe_type], 933 kNumFlows, kOneWayDelayMs); 934 metric_recorders[i].get()->PlotLossHistogram( 935 title, bwe_names[bwe_type], kNumFlows, 936 receivers[i].get()->GlobalPacketLoss()); 937 BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, i); 938 } 939 } 940 941 // Following functions are used for randomizing TCP file size and 942 // starting time, used on 5.7 RunMultipleShortTcpFairness. 943 // They are pseudo-random generators, creating always the same 944 // value sequence for a given Random seed. 945 946 std::vector<int> BweTest::GetFileSizesBytes(int num_files) { 947 // File size chosen from uniform distribution between [100,1000] kB. 948 const int kMinKbytes = 100; 949 const int kMaxKbytes = 1000; 950 951 Random random(0x12345678); 952 std::vector<int> tcp_file_sizes_bytes; 953 954 while (num_files-- > 0) { 955 tcp_file_sizes_bytes.push_back(random.Rand(kMinKbytes, kMaxKbytes) * 1000); 956 } 957 958 return tcp_file_sizes_bytes; 959 } 960 961 std::vector<int64_t> BweTest::GetStartingTimesMs(int num_files) { 962 // OFF state behaves as an exp. distribution with mean = 10 seconds. 963 const float kMeanMs = 10000.0f; 964 Random random(0x12345678); 965 966 std::vector<int64_t> tcp_starting_times_ms; 967 968 // Two TCP Flows are initialized simultaneosly at t=0 seconds. 969 for (int i = 0; i < 2; ++i, --num_files) { 970 tcp_starting_times_ms.push_back(0); 971 } 972 973 // Other TCP Flows are initialized in an OFF state. 974 while (num_files-- > 0) { 975 tcp_starting_times_ms.push_back( 976 static_cast<int64_t>(random.Exponential(1.0f / kMeanMs))); 977 } 978 979 return tcp_starting_times_ms; 980 } 981 982 } // namespace bwe 983 } // namespace testing 984 } // namespace webrtc 985
