Home | History | Annotate | Download | only in test 
      1 /*
      2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #ifndef WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_FRAMEWORK_H_
     12 #define WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_FRAMEWORK_H_
     13 
     14 #include <assert.h>
     15 #include <math.h>
     16 
     17 #include <algorithm>
     18 #include <list>
     19 #include <numeric>
     20 #include <set>
     21 #include <sstream>
     22 #include <string>
     23 #include <utility>
     24 #include <vector>
     25 
     26 #include "webrtc/base/common.h"
     27 #include "webrtc/base/random.h"
     28 #include "webrtc/base/scoped_ptr.h"
     29 #include "webrtc/modules/bitrate_controller/include/bitrate_controller.h"
     30 #include "webrtc/modules/include/module_common_types.h"
     31 #include "webrtc/modules/pacing/paced_sender.h"
     32 #include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
     33 #include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.h"
     34 #include "webrtc/modules/remote_bitrate_estimator/test/packet.h"
     35 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
     36 #include "webrtc/system_wrappers/include/clock.h"
     37 
     38 namespace webrtc {
     39 
     40 class RtcpBandwidthObserver;
     41 
     42 namespace testing {
     43 namespace bwe {
     44 
     45 class DelayCapHelper;
     46 
     47 class RateCounter {
     48  public:
     49   explicit RateCounter(int64_t window_size_ms)
     50       : window_size_us_(1000 * window_size_ms),
     51         recently_received_packets_(0),
     52         recently_received_bytes_(0),
     53         last_accumulated_us_(0),
     54         window_() {}
     55 
     56   RateCounter() : RateCounter(1000) {}
     57 
     58   void UpdateRates(int64_t send_time_us, uint32_t payload_size);
     59 
     60   int64_t window_size_ms() const { return (window_size_us_ + 500) / 1000; }
     61   uint32_t packets_per_second() const;
     62   uint32_t bits_per_second() const;
     63 
     64   double BitrateWindowS() const;
     65 
     66  private:
     67   typedef std::pair<int64_t, uint32_t> TimeSizePair;
     68 
     69   int64_t window_size_us_;
     70   uint32_t recently_received_packets_;
     71   uint32_t recently_received_bytes_;
     72   int64_t last_accumulated_us_;
     73   std::list<TimeSizePair> window_;
     74 };
     75 
     76 typedef std::set<int> FlowIds;
     77 const FlowIds CreateFlowIds(const int *flow_ids_array, size_t num_flow_ids);
     78 const FlowIds CreateFlowIdRange(int initial_value, int last_value);
     79 
     80 template <typename T>
     81 bool DereferencingComparator(const T* const& a, const T* const& b) {
     82   assert(a != NULL);
     83   assert(b != NULL);
     84   return *a < *b;
     85 }
     86 
     87 template<typename T> class Stats {
     88  public:
     89   Stats()
     90       : data_(),
     91         last_mean_count_(0),
     92         last_variance_count_(0),
     93         last_minmax_count_(0),
     94         mean_(0),
     95         variance_(0),
     96         min_(0),
     97         max_(0) {
     98   }
     99 
    100   void Push(T data_point) {
    101     data_.push_back(data_point);
    102   }
    103 
    104   T GetMean() {
    105     if (last_mean_count_ != data_.size()) {
    106       last_mean_count_ = data_.size();
    107       mean_ = std::accumulate(data_.begin(), data_.end(), static_cast<T>(0));
    108       assert(last_mean_count_ != 0);
    109       mean_ /= static_cast<T>(last_mean_count_);
    110     }
    111     return mean_;
    112   }
    113   T GetVariance() {
    114     if (last_variance_count_ != data_.size()) {
    115       last_variance_count_ = data_.size();
    116       T mean = GetMean();
    117       variance_ = 0;
    118       for (const auto& sample : data_) {
    119         T diff = (sample - mean);
    120         variance_ += diff * diff;
    121       }
    122       assert(last_variance_count_ != 0);
    123       variance_ /= static_cast<T>(last_variance_count_);
    124     }
    125     return variance_;
    126   }
    127   T GetStdDev() {
    128     return sqrt(static_cast<double>(GetVariance()));
    129   }
    130   T GetMin() {
    131     RefreshMinMax();
    132     return min_;
    133   }
    134   T GetMax() {
    135     RefreshMinMax();
    136     return max_;
    137   }
    138 
    139   std::string AsString() {
    140     std::stringstream ss;
    141     ss << (GetMean() >= 0 ? GetMean() : -1) << ", " <<
    142         (GetStdDev() >= 0 ? GetStdDev() : -1);
    143     return ss.str();
    144   }
    145 
    146   void Log(const std::string& units) {
    147     BWE_TEST_LOGGING_LOG5("", "%f %s\t+/-%f\t[%f,%f]",
    148         GetMean(), units.c_str(), GetStdDev(), GetMin(), GetMax());
    149   }
    150 
    151  private:
    152   void RefreshMinMax() {
    153     if (last_minmax_count_ != data_.size()) {
    154       last_minmax_count_ = data_.size();
    155       min_ = max_ = 0;
    156       if (data_.empty()) {
    157         return;
    158       }
    159       typename std::vector<T>::const_iterator it = data_.begin();
    160       min_ = max_ = *it;
    161       while (++it != data_.end()) {
    162         min_ = std::min(min_, *it);
    163         max_ = std::max(max_, *it);
    164       }
    165     }
    166   }
    167 
    168   std::vector<T> data_;
    169   typename std::vector<T>::size_type last_mean_count_;
    170   typename std::vector<T>::size_type last_variance_count_;
    171   typename std::vector<T>::size_type last_minmax_count_;
    172   T mean_;
    173   T variance_;
    174   T min_;
    175   T max_;
    176 };
    177 
    178 bool IsTimeSorted(const Packets& packets);
    179 
    180 class PacketProcessor;
    181 
    182 enum ProcessorType { kSender, kReceiver, kRegular };
    183 
    184 class PacketProcessorListener {
    185  public:
    186   virtual ~PacketProcessorListener() {}
    187 
    188   virtual void AddPacketProcessor(PacketProcessor* processor,
    189                                   ProcessorType type) = 0;
    190   virtual void RemovePacketProcessor(PacketProcessor* processor) = 0;
    191 };
    192 
    193 class PacketProcessor {
    194  public:
    195   PacketProcessor(PacketProcessorListener* listener,
    196                   int flow_id,
    197                   ProcessorType type);
    198   PacketProcessor(PacketProcessorListener* listener,
    199                   const FlowIds& flow_ids,
    200                   ProcessorType type);
    201   virtual ~PacketProcessor();
    202 
    203   // Called after each simulation batch to allow the processor to plot any
    204   // internal data.
    205   virtual void Plot(int64_t timestamp_ms) {}
    206 
    207   // Run simulation for |time_ms| milliseconds, consuming packets from, and
    208   // producing packets into in_out. The outgoing packet list must be sorted on
    209   // |send_time_us_|. The simulation time |time_ms| is optional to use.
    210   virtual void RunFor(int64_t time_ms, Packets* in_out) = 0;
    211 
    212   const FlowIds& flow_ids() const { return flow_ids_; }
    213 
    214   uint32_t packets_per_second() const;
    215   uint32_t bits_per_second() const;
    216 
    217  protected:
    218   RateCounter rate_counter_;
    219 
    220  private:
    221   PacketProcessorListener* listener_;
    222   const FlowIds flow_ids_;
    223 
    224   RTC_DISALLOW_COPY_AND_ASSIGN(PacketProcessor);
    225 };
    226 
    227 class RateCounterFilter : public PacketProcessor {
    228  public:
    229   RateCounterFilter(PacketProcessorListener* listener,
    230                     int flow_id,
    231                     const char* name,
    232                     const std::string& plot_name);
    233   RateCounterFilter(PacketProcessorListener* listener,
    234                     const FlowIds& flow_ids,
    235                     const char* name,
    236                     const std::string& plot_name);
    237   RateCounterFilter(PacketProcessorListener* listener,
    238                     const FlowIds& flow_ids,
    239                     const char* name,
    240                     int64_t start_plotting_time_ms,
    241                     const std::string& plot_name);
    242   virtual ~RateCounterFilter();
    243 
    244   void LogStats();
    245   Stats<double> GetBitrateStats() const;
    246   virtual void Plot(int64_t timestamp_ms);
    247   virtual void RunFor(int64_t time_ms, Packets* in_out);
    248 
    249  private:
    250   Stats<double> packets_per_second_stats_;
    251   Stats<double> kbps_stats_;
    252   std::string name_;
    253   int64_t start_plotting_time_ms_;
    254   // Algorithm name if single flow, Total link utilization if all flows.
    255   std::string plot_name_;
    256 
    257   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RateCounterFilter);
    258 };
    259 
    260 class LossFilter : public PacketProcessor {
    261  public:
    262   LossFilter(PacketProcessorListener* listener, int flow_id);
    263   LossFilter(PacketProcessorListener* listener, const FlowIds& flow_ids);
    264   virtual ~LossFilter() {}
    265 
    266   void SetLoss(float loss_percent);
    267   virtual void RunFor(int64_t time_ms, Packets* in_out);
    268 
    269  private:
    270   Random random_;
    271   float loss_fraction_;
    272 
    273   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(LossFilter);
    274 };
    275 
    276 class DelayFilter : public PacketProcessor {
    277  public:
    278   DelayFilter(PacketProcessorListener* listener, int flow_id);
    279   DelayFilter(PacketProcessorListener* listener, const FlowIds& flow_ids);
    280   virtual ~DelayFilter() {}
    281 
    282   void SetOneWayDelayMs(int64_t one_way_delay_ms);
    283   virtual void RunFor(int64_t time_ms, Packets* in_out);
    284 
    285  private:
    286   int64_t one_way_delay_us_;
    287   int64_t last_send_time_us_;
    288 
    289   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(DelayFilter);
    290 };
    291 
    292 class JitterFilter : public PacketProcessor {
    293  public:
    294   JitterFilter(PacketProcessorListener* listener, int flow_id);
    295   JitterFilter(PacketProcessorListener* listener, const FlowIds& flow_ids);
    296   virtual ~JitterFilter() {}
    297 
    298   void SetMaxJitter(int64_t stddev_jitter_ms);
    299   virtual void RunFor(int64_t time_ms, Packets* in_out);
    300   void set_reorderdering(bool reordering) { reordering_ = reordering; }
    301   int64_t MeanUs();
    302 
    303  private:
    304   Random random_;
    305   int64_t stddev_jitter_us_;
    306   int64_t last_send_time_us_;
    307   bool reordering_;  // False by default.
    308 
    309   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(JitterFilter);
    310 };
    311 
    312 // Reorders two consecutive packets with a probability of reorder_percent.
    313 class ReorderFilter : public PacketProcessor {
    314  public:
    315   ReorderFilter(PacketProcessorListener* listener, int flow_id);
    316   ReorderFilter(PacketProcessorListener* listener, const FlowIds& flow_ids);
    317   virtual ~ReorderFilter() {}
    318 
    319   void SetReorder(float reorder_percent);
    320   virtual void RunFor(int64_t time_ms, Packets* in_out);
    321 
    322  private:
    323   Random random_;
    324   float reorder_fraction_;
    325 
    326   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(ReorderFilter);
    327 };
    328 
    329 // Apply a bitrate choke with an infinite queue on the packet stream.
    330 class ChokeFilter : public PacketProcessor {
    331  public:
    332   ChokeFilter(PacketProcessorListener* listener, int flow_id);
    333   ChokeFilter(PacketProcessorListener* listener, const FlowIds& flow_ids);
    334   virtual ~ChokeFilter();
    335 
    336   void set_capacity_kbps(uint32_t kbps);
    337   void set_max_delay_ms(int64_t max_queueing_delay_ms);
    338 
    339   uint32_t capacity_kbps();
    340 
    341   virtual void RunFor(int64_t time_ms, Packets* in_out);
    342 
    343   Stats<double> GetDelayStats() const;
    344 
    345  private:
    346   uint32_t capacity_kbps_;
    347   int64_t last_send_time_us_;
    348   rtc::scoped_ptr<DelayCapHelper> delay_cap_helper_;
    349 
    350   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(ChokeFilter);
    351 };
    352 
    353 class TraceBasedDeliveryFilter : public PacketProcessor {
    354  public:
    355   TraceBasedDeliveryFilter(PacketProcessorListener* listener, int flow_id);
    356   TraceBasedDeliveryFilter(PacketProcessorListener* listener,
    357                            const FlowIds& flow_ids);
    358   TraceBasedDeliveryFilter(PacketProcessorListener* listener,
    359                            int flow_id,
    360                            const char* name);
    361   virtual ~TraceBasedDeliveryFilter();
    362 
    363   // The file should contain nanosecond timestamps corresponding to the time
    364   // when the network can accept another packet. The timestamps should be
    365   // separated by new lines, e.g., "100000000\n125000000\n321000000\n..."
    366   bool Init(const std::string& filename);
    367   virtual void Plot(int64_t timestamp_ms);
    368   virtual void RunFor(int64_t time_ms, Packets* in_out);
    369 
    370   void set_max_delay_ms(int64_t max_delay_ms);
    371   Stats<double> GetDelayStats() const;
    372   Stats<double> GetBitrateStats() const;
    373 
    374  private:
    375   void ProceedToNextSlot();
    376 
    377   typedef std::vector<int64_t> TimeList;
    378   int64_t current_offset_us_;
    379   TimeList delivery_times_us_;
    380   TimeList::const_iterator next_delivery_it_;
    381   int64_t local_time_us_;
    382   rtc::scoped_ptr<RateCounter> rate_counter_;
    383   std::string name_;
    384   rtc::scoped_ptr<DelayCapHelper> delay_cap_helper_;
    385   Stats<double> packets_per_second_stats_;
    386   Stats<double> kbps_stats_;
    387 
    388   RTC_DISALLOW_COPY_AND_ASSIGN(TraceBasedDeliveryFilter);
    389 };
    390 
    391 class VideoSource {
    392  public:
    393   VideoSource(int flow_id,
    394               float fps,
    395               uint32_t kbps,
    396               uint32_t ssrc,
    397               int64_t first_frame_offset_ms);
    398   virtual ~VideoSource() {}
    399 
    400   virtual void RunFor(int64_t time_ms, Packets* in_out);
    401 
    402   virtual int flow_id() const { return flow_id_; }
    403   virtual void SetBitrateBps(int bitrate_bps) {}
    404   uint32_t bits_per_second() const { return bits_per_second_; }
    405   uint32_t max_payload_size_bytes() const { return kMaxPayloadSizeBytes; }
    406   int64_t GetTimeUntilNextFrameMs() const;
    407 
    408  protected:
    409   virtual uint32_t NextFrameSize();
    410   virtual uint32_t NextPacketSize(uint32_t frame_size,
    411                                   uint32_t remaining_payload);
    412 
    413   const uint32_t kMaxPayloadSizeBytes;
    414   const uint32_t kTimestampBase;
    415   const double frame_period_ms_;
    416   uint32_t bits_per_second_;
    417   uint32_t frame_size_bytes_;
    418 
    419  private:
    420   Random random_;
    421   const int flow_id_;
    422   int64_t next_frame_ms_;
    423   int64_t next_frame_rand_ms_;
    424   int64_t now_ms_;
    425   RTPHeader prototype_header_;
    426 
    427   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(VideoSource);
    428 };
    429 
    430 class AdaptiveVideoSource : public VideoSource {
    431  public:
    432   AdaptiveVideoSource(int flow_id,
    433                       float fps,
    434                       uint32_t kbps,
    435                       uint32_t ssrc,
    436                       int64_t first_frame_offset_ms);
    437   virtual ~AdaptiveVideoSource() {}
    438 
    439   void SetBitrateBps(int bitrate_bps) override;
    440 
    441  private:
    442   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AdaptiveVideoSource);
    443 };
    444 
    445 class PeriodicKeyFrameSource : public AdaptiveVideoSource {
    446  public:
    447   PeriodicKeyFrameSource(int flow_id,
    448                          float fps,
    449                          uint32_t kbps,
    450                          uint32_t ssrc,
    451                          int64_t first_frame_offset_ms,
    452                          int key_frame_interval);
    453   virtual ~PeriodicKeyFrameSource() {}
    454 
    455  protected:
    456   uint32_t NextFrameSize() override;
    457   uint32_t NextPacketSize(uint32_t frame_size,
    458                           uint32_t remaining_payload) override;
    459 
    460  private:
    461   int key_frame_interval_;
    462   uint32_t frame_counter_;
    463   int compensation_bytes_;
    464   int compensation_per_frame_;
    465   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(PeriodicKeyFrameSource);
    466 };
    467 }  // namespace bwe
    468 }  // namespace testing
    469 }  // namespace webrtc
    470 
    471 #endif  // WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_FRAMEWORK_H_
    472