Home | History | Annotate | Download | only in test 
      1 /*
      2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #ifndef WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_FRAMEWORK_H_
     12 #define WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_FRAMEWORK_H_
     13 
     14 #include <assert.h>
     15 #include <math.h>
     16 
     17 #include <algorithm>
     18 #include <list>
     19 #include <numeric>
     20 #include <sstream>
     21 #include <string>
     22 #include <vector>
     23 
     24 #include "webrtc/modules/interface/module_common_types.h"
     25 #include "webrtc/modules/pacing/include/paced_sender.h"
     26 #include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.h"
     27 #include "webrtc/system_wrappers/interface/clock.h"
     28 #include "webrtc/system_wrappers/interface/scoped_ptr.h"
     29 
     30 namespace webrtc {
     31 namespace testing {
     32 namespace bwe {
     33 
     34 class DelayCapHelper;
     35 class RateCounter;
     36 
     37 
     38 typedef std::vector<int> FlowIds;
     39 const FlowIds CreateFlowIds(const int *flow_ids_array, size_t num_flow_ids);
     40 
     41 template<typename T> class Stats {
     42  public:
     43   Stats()
     44       : data_(),
     45         last_mean_count_(0),
     46         last_variance_count_(0),
     47         last_minmax_count_(0),
     48         mean_(0),
     49         variance_(0),
     50         min_(0),
     51         max_(0) {
     52   }
     53 
     54   void Push(T data_point) {
     55     data_.push_back(data_point);
     56   }
     57 
     58   T GetMean() {
     59     if (last_mean_count_ != data_.size()) {
     60       last_mean_count_ = data_.size();
     61       mean_ = std::accumulate(data_.begin(), data_.end(), static_cast<T>(0));
     62       assert(last_mean_count_ != 0);
     63       mean_ /= static_cast<T>(last_mean_count_);
     64     }
     65     return mean_;
     66   }
     67   T GetVariance() {
     68     if (last_variance_count_ != data_.size()) {
     69       last_variance_count_ = data_.size();
     70       T mean = GetMean();
     71       variance_ = 0;
     72       for (typename std::vector<T>::const_iterator it = data_.begin();
     73           it != data_.end(); ++it) {
     74         T diff = (*it - mean);
     75         variance_ += diff * diff;
     76       }
     77       assert(last_variance_count_ != 0);
     78       variance_ /= static_cast<T>(last_variance_count_);
     79     }
     80     return variance_;
     81   }
     82   T GetStdDev() {
     83     return sqrt(static_cast<double>(GetVariance()));
     84   }
     85   T GetMin() {
     86     RefreshMinMax();
     87     return min_;
     88   }
     89   T GetMax() {
     90     RefreshMinMax();
     91     return max_;
     92   }
     93 
     94   std::string AsString() {
     95     std::stringstream ss;
     96     ss << (GetMean() >= 0 ? GetMean() : -1) << ", " <<
     97         (GetStdDev() >= 0 ? GetStdDev() : -1);
     98     return ss.str();
     99   }
    100 
    101   void Log(const std::string& units) {
    102     BWE_TEST_LOGGING_LOG5("", "%f %s\t+/-%f\t[%f,%f]",
    103         GetMean(), units.c_str(), GetStdDev(), GetMin(), GetMax());
    104   }
    105 
    106  private:
    107   void RefreshMinMax() {
    108     if (last_minmax_count_ != data_.size()) {
    109       last_minmax_count_ = data_.size();
    110       min_ = max_ = 0;
    111       if (data_.empty()) {
    112         return;
    113       }
    114       typename std::vector<T>::const_iterator it = data_.begin();
    115       min_ = max_ = *it;
    116       while (++it != data_.end()) {
    117         min_ = std::min(min_, *it);
    118         max_ = std::max(max_, *it);
    119       }
    120     }
    121   }
    122 
    123   std::vector<T> data_;
    124   typename std::vector<T>::size_type last_mean_count_;
    125   typename std::vector<T>::size_type last_variance_count_;
    126   typename std::vector<T>::size_type last_minmax_count_;
    127   T mean_;
    128   T variance_;
    129   T min_;
    130   T max_;
    131 };
    132 
    133 class Random {
    134  public:
    135   explicit Random(uint32_t seed);
    136 
    137   // Return pseudo random number in the interval [0.0, 1.0].
    138   float Rand();
    139 
    140   // Normal Distribution.
    141   int Gaussian(int mean, int standard_deviation);
    142 
    143   // TODO(solenberg): Random from histogram.
    144   // template<typename T> int Distribution(const std::vector<T> histogram) {
    145 
    146  private:
    147   uint32_t a_;
    148   uint32_t b_;
    149 
    150   DISALLOW_IMPLICIT_CONSTRUCTORS(Random);
    151 };
    152 
    153 class Packet {
    154  public:
    155   Packet();
    156   Packet(int flow_id, int64_t send_time_us, uint32_t payload_size,
    157          const RTPHeader& header);
    158   Packet(int64_t send_time_us, uint32_t sequence_number);
    159 
    160   bool operator<(const Packet& rhs) const;
    161 
    162   int flow_id() const { return flow_id_; }
    163   int64_t creation_time_us() const { return creation_time_us_; }
    164   void set_send_time_us(int64_t send_time_us);
    165   int64_t send_time_us() const { return send_time_us_; }
    166   void SetAbsSendTimeMs(int64_t abs_send_time_ms);
    167   uint32_t payload_size() const { return payload_size_; }
    168   const RTPHeader& header() const { return header_; }
    169 
    170  private:
    171   int flow_id_;
    172   int64_t creation_time_us_;  // Time when the packet was created.
    173   int64_t send_time_us_;   // Time the packet left last processor touching it.
    174   uint32_t payload_size_;  // Size of the (non-existent, simulated) payload.
    175   RTPHeader header_;       // Actual contents.
    176 };
    177 
    178 typedef std::list<Packet> Packets;
    179 typedef std::list<Packet>::iterator PacketsIt;
    180 typedef std::list<Packet>::const_iterator PacketsConstIt;
    181 
    182 bool IsTimeSorted(const Packets& packets);
    183 
    184 class PacketProcessor;
    185 
    186 class PacketProcessorListener {
    187  public:
    188   virtual ~PacketProcessorListener() {}
    189 
    190   virtual void AddPacketProcessor(PacketProcessor* processor,
    191                                   bool is_sender) = 0;
    192   virtual void RemovePacketProcessor(PacketProcessor* processor) = 0;
    193 };
    194 
    195 class PacketProcessor {
    196  public:
    197   PacketProcessor(PacketProcessorListener* listener, bool is_sender);
    198   PacketProcessor(PacketProcessorListener* listener, const FlowIds& flow_ids,
    199                   bool is_sender);
    200   virtual ~PacketProcessor();
    201 
    202   // Called after each simulation batch to allow the processor to plot any
    203   // internal data.
    204   virtual void Plot(int64_t timestamp_ms) {}
    205 
    206   // Run simulation for |time_ms| micro seconds, consuming packets from, and
    207   // producing packets into in_out. The outgoing packet list must be sorted on
    208   // |send_time_us_|. The simulation time |time_ms| is optional to use.
    209   virtual void RunFor(int64_t time_ms, Packets* in_out) = 0;
    210 
    211   const FlowIds& flow_ids() const { return flow_ids_; }
    212 
    213  private:
    214   PacketProcessorListener* listener_;
    215   FlowIds flow_ids_;
    216 
    217   DISALLOW_COPY_AND_ASSIGN(PacketProcessor);
    218 };
    219 
    220 class RateCounterFilter : public PacketProcessor {
    221  public:
    222   explicit RateCounterFilter(PacketProcessorListener* listener);
    223   RateCounterFilter(PacketProcessorListener* listener,
    224                     const std::string& name);
    225   RateCounterFilter(PacketProcessorListener* listener,
    226                     const FlowIds& flow_ids,
    227                     const std::string& name);
    228   virtual ~RateCounterFilter();
    229 
    230   uint32_t packets_per_second() const;
    231   uint32_t bits_per_second() const;
    232 
    233   void LogStats();
    234   Stats<double> GetBitrateStats() const;
    235   virtual void Plot(int64_t timestamp_ms);
    236   virtual void RunFor(int64_t time_ms, Packets* in_out);
    237 
    238  private:
    239   scoped_ptr<RateCounter> rate_counter_;
    240   Stats<double> packets_per_second_stats_;
    241   Stats<double> kbps_stats_;
    242   std::string name_;
    243 
    244   DISALLOW_IMPLICIT_CONSTRUCTORS(RateCounterFilter);
    245 };
    246 
    247 class LossFilter : public PacketProcessor {
    248  public:
    249   explicit LossFilter(PacketProcessorListener* listener);
    250   virtual ~LossFilter() {}
    251 
    252   void SetLoss(float loss_percent);
    253   virtual void RunFor(int64_t time_ms, Packets* in_out);
    254 
    255  private:
    256   Random random_;
    257   float loss_fraction_;
    258 
    259   DISALLOW_IMPLICIT_CONSTRUCTORS(LossFilter);
    260 };
    261 
    262 class DelayFilter : public PacketProcessor {
    263  public:
    264   explicit DelayFilter(PacketProcessorListener* listener);
    265   virtual ~DelayFilter() {}
    266 
    267   void SetDelay(int64_t delay_ms);
    268   virtual void RunFor(int64_t time_ms, Packets* in_out);
    269 
    270  private:
    271   int64_t delay_us_;
    272   int64_t last_send_time_us_;
    273 
    274   DISALLOW_IMPLICIT_CONSTRUCTORS(DelayFilter);
    275 };
    276 
    277 class JitterFilter : public PacketProcessor {
    278  public:
    279   explicit JitterFilter(PacketProcessorListener* listener);
    280   virtual ~JitterFilter() {}
    281 
    282   void SetJitter(int64_t stddev_jitter_ms);
    283   virtual void RunFor(int64_t time_ms, Packets* in_out);
    284 
    285  private:
    286   Random random_;
    287   int64_t stddev_jitter_us_;
    288   int64_t last_send_time_us_;
    289 
    290   DISALLOW_IMPLICIT_CONSTRUCTORS(JitterFilter);
    291 };
    292 
    293 class ReorderFilter : public PacketProcessor {
    294  public:
    295   explicit ReorderFilter(PacketProcessorListener* listener);
    296   virtual ~ReorderFilter() {}
    297 
    298   void SetReorder(float reorder_percent);
    299   virtual void RunFor(int64_t time_ms, Packets* in_out);
    300 
    301  private:
    302   Random random_;
    303   float reorder_fraction_;
    304 
    305   DISALLOW_IMPLICIT_CONSTRUCTORS(ReorderFilter);
    306 };
    307 
    308 // Apply a bitrate choke with an infinite queue on the packet stream.
    309 class ChokeFilter : public PacketProcessor {
    310  public:
    311   explicit ChokeFilter(PacketProcessorListener* listener);
    312   ChokeFilter(PacketProcessorListener* listener, const FlowIds& flow_ids);
    313   virtual ~ChokeFilter();
    314 
    315   void SetCapacity(uint32_t kbps);
    316   void SetMaxDelay(int max_delay_ms);
    317   virtual void RunFor(int64_t time_ms, Packets* in_out);
    318 
    319   Stats<double> GetDelayStats() const;
    320 
    321  private:
    322   uint32_t kbps_;
    323   int64_t last_send_time_us_;
    324   scoped_ptr<DelayCapHelper> delay_cap_helper_;
    325 
    326   DISALLOW_IMPLICIT_CONSTRUCTORS(ChokeFilter);
    327 };
    328 
    329 class TraceBasedDeliveryFilter : public PacketProcessor {
    330  public:
    331   explicit TraceBasedDeliveryFilter(PacketProcessorListener* listener);
    332   TraceBasedDeliveryFilter(PacketProcessorListener* listener,
    333                            const std::string& name);
    334   virtual ~TraceBasedDeliveryFilter();
    335 
    336   // The file should contain nanosecond timestamps corresponding to the time
    337   // when the network can accept another packet. The timestamps should be
    338   // separated by new lines, e.g., "100000000\n125000000\n321000000\n..."
    339   bool Init(const std::string& filename);
    340   virtual void Plot(int64_t timestamp_ms);
    341   virtual void RunFor(int64_t time_ms, Packets* in_out);
    342 
    343   void SetMaxDelay(int max_delay_ms);
    344   Stats<double> GetDelayStats() const;
    345   Stats<double> GetBitrateStats() const;
    346 
    347  private:
    348   void ProceedToNextSlot();
    349 
    350   typedef std::vector<int64_t> TimeList;
    351   int64_t current_offset_us_;
    352   TimeList delivery_times_us_;
    353   TimeList::const_iterator next_delivery_it_;
    354   int64_t local_time_us_;
    355   scoped_ptr<RateCounter> rate_counter_;
    356   std::string name_;
    357   scoped_ptr<DelayCapHelper> delay_cap_helper_;
    358   Stats<double> packets_per_second_stats_;
    359   Stats<double> kbps_stats_;
    360 
    361   DISALLOW_COPY_AND_ASSIGN(TraceBasedDeliveryFilter);
    362 };
    363 
    364 class PacketSender : public PacketProcessor {
    365  public:
    366   struct Feedback {
    367     uint32_t estimated_bps;
    368   };
    369 
    370   explicit PacketSender(PacketProcessorListener* listener);
    371   PacketSender(PacketProcessorListener* listener, const FlowIds& flow_ids);
    372   virtual ~PacketSender() {}
    373 
    374   virtual uint32_t GetCapacityKbps() const { return 0; }
    375 
    376   // Call GiveFeedback() with the returned interval in milliseconds, provided
    377   // there is a new estimate available.
    378   // Note that changing the feedback interval affects the timing of when the
    379   // output of the estimators is sampled and therefore the baseline files may
    380   // have to be regenerated.
    381   virtual int GetFeedbackIntervalMs() const { return 1000; }
    382   virtual void GiveFeedback(const Feedback& feedback) {}
    383 
    384  private:
    385   DISALLOW_COPY_AND_ASSIGN(PacketSender);
    386 };
    387 
    388 class VideoSender : public PacketSender {
    389  public:
    390   VideoSender(int flow_id, PacketProcessorListener* listener, float fps,
    391               uint32_t kbps, uint32_t ssrc, float first_frame_offset);
    392   virtual ~VideoSender() {}
    393 
    394   uint32_t max_payload_size_bytes() const { return kMaxPayloadSizeBytes; }
    395   uint32_t bytes_per_second() const { return bytes_per_second_; }
    396 
    397   virtual uint32_t GetCapacityKbps() const OVERRIDE;
    398 
    399   virtual void RunFor(int64_t time_ms, Packets* in_out) OVERRIDE;
    400 
    401  protected:
    402   const uint32_t kMaxPayloadSizeBytes;
    403   const uint32_t kTimestampBase;
    404   const double frame_period_ms_;
    405   uint32_t bytes_per_second_;
    406   uint32_t frame_size_bytes_;
    407 
    408  private:
    409   double next_frame_ms_;
    410   double now_ms_;
    411   RTPHeader prototype_header_;
    412 
    413   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoSender);
    414 };
    415 
    416 class AdaptiveVideoSender : public VideoSender {
    417  public:
    418   AdaptiveVideoSender(int flow_id, PacketProcessorListener* listener,
    419                       float fps, uint32_t kbps, uint32_t ssrc,
    420                       float first_frame_offset);
    421   virtual ~AdaptiveVideoSender() {}
    422 
    423   virtual int GetFeedbackIntervalMs() const OVERRIDE { return 100; }
    424   virtual void GiveFeedback(const Feedback& feedback) OVERRIDE;
    425 
    426  private:
    427   DISALLOW_IMPLICIT_CONSTRUCTORS(AdaptiveVideoSender);
    428 };
    429 
    430 class PacedVideoSender : public PacketSender, public PacedSender::Callback {
    431  public:
    432   PacedVideoSender(PacketProcessorListener* listener,
    433                    uint32_t kbps, AdaptiveVideoSender* source);
    434   virtual ~PacedVideoSender() {}
    435 
    436   virtual int GetFeedbackIntervalMs() const OVERRIDE { return 100; }
    437   virtual void GiveFeedback(const Feedback& feedback) OVERRIDE;
    438   virtual void RunFor(int64_t time_ms, Packets* in_out) OVERRIDE;
    439 
    440   // Implements PacedSender::Callback.
    441   virtual bool TimeToSendPacket(uint32_t ssrc,
    442                                 uint16_t sequence_number,
    443                                 int64_t capture_time_ms,
    444                                 bool retransmission) OVERRIDE;
    445   virtual int TimeToSendPadding(int bytes) OVERRIDE;
    446 
    447  private:
    448   void QueuePackets(Packets* batch, int64_t end_of_batch_time_us);
    449 
    450   static const int64_t kInitialTimeMs = 0;
    451   SimulatedClock clock_;
    452   int64_t start_of_run_ms_;
    453   PacedSender pacer_;
    454   Packets pacer_queue_;
    455   Packets queue_;
    456   AdaptiveVideoSender* source_;
    457 
    458   DISALLOW_IMPLICIT_CONSTRUCTORS(PacedVideoSender);
    459 };
    460 }  // namespace bwe
    461 }  // namespace testing
    462 }  // namespace webrtc
    463 
    464 #endif  // WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_FRAMEWORK_H_
    465