1 // libjingle 2 // Copyright 2010 Google Inc. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are met: 6 // 7 // 1. Redistributions of source code must retain the above copyright notice, 8 // this list of conditions and the following disclaimer. 9 // 2. Redistributions in binary form must reproduce the above copyright notice, 10 // this list of conditions and the following disclaimer in the documentation 11 // and/or other materials provided with the distribution. 12 // 3. The name of the author may not be used to endorse or promote products 13 // derived from this software without specific prior written permission. 14 // 15 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 16 // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 17 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 18 // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 19 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 24 // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 26 #ifndef TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT 27 #define TALK_MEDIA_BASE_VIDEOADAPTER_H_ 28 29 #include "talk/base/common.h" // For ASSERT 30 #include "talk/base/criticalsection.h" 31 #include "talk/base/logging.h" 32 #include "talk/base/scoped_ptr.h" 33 #include "talk/base/sigslot.h" 34 #include "talk/media/base/videocommon.h" 35 36 namespace cricket { 37 38 class VideoFrame; 39 40 // VideoAdapter adapts an input video frame to an output frame based on the 41 // specified input and output formats. The adaptation includes dropping frames 42 // to reduce frame rate and scaling frames. VideoAdapter is thread safe. 43 class VideoAdapter { 44 public: 45 VideoAdapter(); 46 virtual ~VideoAdapter(); 47 48 void SetInputFormat(const VideoFrame& in_frame); 49 void SetInputFormat(const VideoFormat& format); 50 void SetOutputFormat(const VideoFormat& format); 51 // Constrain output resolution to this many pixels overall 52 void SetOutputNumPixels(int num_pixels); 53 int GetOutputNumPixels() const; 54 55 const VideoFormat& input_format(); 56 const VideoFormat& output_format(); 57 // If the parameter black is true, the adapted frames will be black. 58 void SetBlackOutput(bool black); 59 60 // Adapt the input frame from the input format to the output format. Return 61 // true and set the output frame to NULL if the input frame is dropped. Return 62 // true and set the out frame to output_frame_ if the input frame is adapted 63 // successfully. Return false otherwise. 64 // output_frame_ is owned by the VideoAdapter that has the best knowledge on 65 // the output frame. 66 bool AdaptFrame(const VideoFrame* in_frame, const VideoFrame** out_frame); 67 68 protected: 69 float FindClosestScale(int width, int height, int target_num_pixels); 70 float FindLowerScale(int width, int height, int target_num_pixels); 71 72 private: 73 bool StretchToOutputFrame(const VideoFrame* in_frame); 74 75 VideoFormat input_format_; 76 VideoFormat output_format_; 77 int output_num_pixels_; 78 bool black_output_; // Flag to tell if we need to black output_frame_. 79 bool is_black_; // Flag to tell if output_frame_ is currently black. 80 int64 interval_next_frame_; 81 talk_base::scoped_ptr<VideoFrame> output_frame_; 82 // The critical section to protect the above variables. 83 talk_base::CriticalSection critical_section_; 84 85 DISALLOW_COPY_AND_ASSIGN(VideoAdapter); 86 }; 87 88 // CoordinatedVideoAdapter adapts the video input to the encoder by coordinating 89 // the format request from the server, the resolution request from the encoder, 90 // and the CPU load. 91 class CoordinatedVideoAdapter 92 : public VideoAdapter, public sigslot::has_slots<> { 93 public: 94 enum AdaptRequest { UPGRADE, KEEP, DOWNGRADE }; 95 enum { 96 ADAPTREASON_CPU = 1, 97 ADAPTREASON_BANDWIDTH = 2, 98 ADAPTREASON_VIEW = 4 99 }; 100 typedef int AdaptReason; 101 102 CoordinatedVideoAdapter(); 103 virtual ~CoordinatedVideoAdapter() {} 104 105 // Enable or disable video adaptation due to the change of the CPU load. 106 void set_cpu_adaptation(bool enable) { cpu_adaptation_ = enable; } 107 bool cpu_adaptation() const { return cpu_adaptation_; } 108 // Enable or disable smoothing when doing CPU adaptation. When smoothing is 109 // enabled, system CPU load is tracked using an exponential weighted 110 // average. 111 void set_cpu_smoothing(bool enable) { 112 LOG(LS_INFO) << "CPU smoothing is now " 113 << (enable ? "enabled" : "disabled"); 114 cpu_smoothing_ = enable; 115 } 116 bool cpu_smoothing() const { return cpu_smoothing_; } 117 // Enable or disable video adaptation due to the change of the GD 118 void set_gd_adaptation(bool enable) { gd_adaptation_ = enable; } 119 bool gd_adaptation() const { return gd_adaptation_; } 120 // Enable or disable video adaptation due to the change of the View 121 void set_view_adaptation(bool enable) { view_adaptation_ = enable; } 122 bool view_adaptation() const { return view_adaptation_; } 123 // Enable or disable video adaptation to fast switch View 124 void set_view_switch(bool enable) { view_switch_ = enable; } 125 bool view_switch() const { return view_switch_; } 126 127 CoordinatedVideoAdapter::AdaptReason adapt_reason() const { 128 return adapt_reason_; 129 } 130 131 // When the video is decreased, set the waiting time for CPU adaptation to 132 // decrease video again. 133 void set_cpu_adapt_wait_time(uint32 cpu_adapt_wait_time) { 134 if (cpu_adapt_wait_time_ != static_cast<int>(cpu_adapt_wait_time)) { 135 LOG(LS_INFO) << "VAdapt Change Cpu Adapt Wait Time from: " 136 << cpu_adapt_wait_time_ << " to " 137 << cpu_adapt_wait_time; 138 cpu_adapt_wait_time_ = static_cast<int>(cpu_adapt_wait_time); 139 } 140 } 141 // CPU system load high threshold for reducing resolution. e.g. 0.85f 142 void set_high_system_threshold(float high_system_threshold) { 143 ASSERT(high_system_threshold <= 1.0f); 144 ASSERT(high_system_threshold >= 0.0f); 145 if (high_system_threshold_ != high_system_threshold) { 146 LOG(LS_INFO) << "VAdapt Change High System Threshold from: " 147 << high_system_threshold_ << " to " << high_system_threshold; 148 high_system_threshold_ = high_system_threshold; 149 } 150 } 151 float high_system_threshold() const { return high_system_threshold_; } 152 // CPU system load low threshold for increasing resolution. e.g. 0.70f 153 void set_low_system_threshold(float low_system_threshold) { 154 ASSERT(low_system_threshold <= 1.0f); 155 ASSERT(low_system_threshold >= 0.0f); 156 if (low_system_threshold_ != low_system_threshold) { 157 LOG(LS_INFO) << "VAdapt Change Low System Threshold from: " 158 << low_system_threshold_ << " to " << low_system_threshold; 159 low_system_threshold_ = low_system_threshold; 160 } 161 } 162 float low_system_threshold() const { return low_system_threshold_; } 163 // CPU process load threshold for reducing resolution. e.g. 0.10f 164 void set_process_threshold(float process_threshold) { 165 ASSERT(process_threshold <= 1.0f); 166 ASSERT(process_threshold >= 0.0f); 167 if (process_threshold_ != process_threshold) { 168 LOG(LS_INFO) << "VAdapt Change High Process Threshold from: " 169 << process_threshold_ << " to " << process_threshold; 170 process_threshold_ = process_threshold; 171 } 172 } 173 float process_threshold() const { return process_threshold_; } 174 175 // Handle the format request from the server via Jingle update message. 176 void OnOutputFormatRequest(const VideoFormat& format); 177 // Handle the resolution request from the encoder due to bandwidth changes. 178 void OnEncoderResolutionRequest(int width, int height, AdaptRequest request); 179 // Handle the CPU load provided by a CPU monitor. 180 void OnCpuLoadUpdated(int current_cpus, int max_cpus, 181 float process_load, float system_load); 182 183 sigslot::signal0<> SignalCpuAdaptationUnable; 184 185 private: 186 // Adapt to the minimum of the formats the server requests, the CPU wants, and 187 // the encoder wants. Returns true if resolution changed. 188 bool AdaptToMinimumFormat(int* new_width, int* new_height); 189 bool IsMinimumFormat(int pixels); 190 void StepPixelCount(CoordinatedVideoAdapter::AdaptRequest request, 191 int* num_pixels); 192 CoordinatedVideoAdapter::AdaptRequest FindCpuRequest( 193 int current_cpus, int max_cpus, 194 float process_load, float system_load); 195 196 bool cpu_adaptation_; // True if cpu adaptation is enabled. 197 bool cpu_smoothing_; // True if cpu smoothing is enabled (with adaptation). 198 bool gd_adaptation_; // True if gd adaptation is enabled. 199 bool view_adaptation_; // True if view adaptation is enabled. 200 bool view_switch_; // True if view switch is enabled. 201 int cpu_downgrade_count_; 202 int cpu_adapt_wait_time_; 203 // cpu system load thresholds relative to max cpus. 204 float high_system_threshold_; 205 float low_system_threshold_; 206 // cpu process load thresholds relative to current cpus. 207 float process_threshold_; 208 // Video formats that the server view requests, the CPU wants, and the encoder 209 // wants respectively. The adapted output format is the minimum of these. 210 int view_desired_num_pixels_; 211 int64 view_desired_interval_; 212 int encoder_desired_num_pixels_; 213 int cpu_desired_num_pixels_; 214 CoordinatedVideoAdapter::AdaptReason adapt_reason_; 215 // The critical section to protect handling requests. 216 talk_base::CriticalSection request_critical_section_; 217 218 // The weighted average of cpu load over time. It's always updated (if cpu 219 // adaptation is on), but only used if cpu_smoothing_ is set. 220 float system_load_average_; 221 222 DISALLOW_COPY_AND_ASSIGN(CoordinatedVideoAdapter); 223 }; 224 225 } // namespace cricket 226 227 #endif // TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT 228
