1 // libjingle 2 // Copyright 2010 Google Inc. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are met: 6 // 7 // 1. Redistributions of source code must retain the above copyright notice, 8 // this list of conditions and the following disclaimer. 9 // 2. Redistributions in binary form must reproduce the above copyright notice, 10 // this list of conditions and the following disclaimer in the documentation 11 // and/or other materials provided with the distribution. 12 // 3. The name of the author may not be used to endorse or promote products 13 // derived from this software without specific prior written permission. 14 // 15 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 16 // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 17 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 18 // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 19 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 24 // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 26 #ifndef TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT 27 #define TALK_MEDIA_BASE_VIDEOADAPTER_H_ 28 29 #include "talk/base/common.h" // For ASSERT 30 #include "talk/base/criticalsection.h" 31 #include "talk/base/logging.h" 32 #include "talk/base/scoped_ptr.h" 33 #include "talk/base/sigslot.h" 34 #include "talk/media/base/videocommon.h" 35 36 namespace cricket { 37 38 class VideoFrame; 39 40 // VideoAdapter adapts an input video frame to an output frame based on the 41 // specified input and output formats. The adaptation includes dropping frames 42 // to reduce frame rate and scaling frames. VideoAdapter is thread safe. 43 class VideoAdapter { 44 public: 45 VideoAdapter(); 46 virtual ~VideoAdapter(); 47 48 virtual void SetInputFormat(const VideoFormat& format); 49 void SetOutputFormat(const VideoFormat& format); 50 // Constrain output resolution to this many pixels overall 51 void SetOutputNumPixels(int num_pixels); 52 int GetOutputNumPixels() const; 53 54 const VideoFormat& input_format(); 55 const VideoFormat& output_format(); 56 // If the parameter black is true, the adapted frames will be black. 57 void SetBlackOutput(bool black); 58 59 // Adapt the input frame from the input format to the output format. Return 60 // true and set the output frame to NULL if the input frame is dropped. Return 61 // true and set the out frame to output_frame_ if the input frame is adapted 62 // successfully. Return false otherwise. 63 // output_frame_ is owned by the VideoAdapter that has the best knowledge on 64 // the output frame. 65 bool AdaptFrame(const VideoFrame* in_frame, VideoFrame** out_frame); 66 67 void set_scale_third(bool enable) { 68 LOG(LS_INFO) << "Video Adapter third scaling is now " 69 << (enable ? "enabled" : "disabled"); 70 scale_third_ = enable; 71 } 72 bool scale_third() const { return scale_third_; } 73 74 protected: 75 float FindClosestScale(int width, int height, int target_num_pixels); 76 float FindClosestViewScale(int width, int height, int target_num_pixels); 77 float FindLowerScale(int width, int height, int target_num_pixels); 78 79 private: 80 const float* GetViewScaleFactors() const; 81 float FindScale(const float* scale_factors, 82 const float upbias, int width, int height, 83 int target_num_pixels); 84 bool StretchToOutputFrame(const VideoFrame* in_frame); 85 86 VideoFormat input_format_; 87 VideoFormat output_format_; 88 int output_num_pixels_; 89 bool scale_third_; // True if adapter allows scaling to 1/3 and 2/3. 90 int frames_; // Number of input frames. 91 int adapted_frames_; // Number of frames scaled. 92 int adaption_changes_; // Number of changes in scale factor. 93 size_t previous_width_; // Previous adapter output width. 94 size_t previous_height_; // Previous adapter output height. 95 bool black_output_; // Flag to tell if we need to black output_frame_. 96 bool is_black_; // Flag to tell if output_frame_ is currently black. 97 int64 interval_next_frame_; 98 talk_base::scoped_ptr<VideoFrame> output_frame_; 99 // The critical section to protect the above variables. 100 talk_base::CriticalSection critical_section_; 101 102 DISALLOW_COPY_AND_ASSIGN(VideoAdapter); 103 }; 104 105 // CoordinatedVideoAdapter adapts the video input to the encoder by coordinating 106 // the format request from the server, the resolution request from the encoder, 107 // and the CPU load. 108 class CoordinatedVideoAdapter 109 : public VideoAdapter, public sigslot::has_slots<> { 110 public: 111 enum AdaptRequest { UPGRADE, KEEP, DOWNGRADE }; 112 enum AdaptReasonEnum { 113 ADAPTREASON_CPU = 1, 114 ADAPTREASON_BANDWIDTH = 2, 115 ADAPTREASON_VIEW = 4 116 }; 117 typedef int AdaptReason; 118 119 CoordinatedVideoAdapter(); 120 virtual ~CoordinatedVideoAdapter() {} 121 122 virtual void SetInputFormat(const VideoFormat& format); 123 124 // Enable or disable video adaptation due to the change of the CPU load. 125 void set_cpu_adaptation(bool enable) { cpu_adaptation_ = enable; } 126 bool cpu_adaptation() const { return cpu_adaptation_; } 127 // Enable or disable smoothing when doing CPU adaptation. When smoothing is 128 // enabled, system CPU load is tracked using an exponential weighted 129 // average. 130 void set_cpu_smoothing(bool enable) { 131 LOG(LS_INFO) << "CPU smoothing is now " 132 << (enable ? "enabled" : "disabled"); 133 cpu_smoothing_ = enable; 134 } 135 bool cpu_smoothing() const { return cpu_smoothing_; } 136 // Enable or disable video adaptation due to the change of the GD 137 void set_gd_adaptation(bool enable) { gd_adaptation_ = enable; } 138 bool gd_adaptation() const { return gd_adaptation_; } 139 // Enable or disable video adaptation due to the change of the View 140 void set_view_adaptation(bool enable) { view_adaptation_ = enable; } 141 bool view_adaptation() const { return view_adaptation_; } 142 // Enable or disable video adaptation to fast switch View 143 void set_view_switch(bool enable) { view_switch_ = enable; } 144 bool view_switch() const { return view_switch_; } 145 146 CoordinatedVideoAdapter::AdaptReason adapt_reason() const { 147 return adapt_reason_; 148 } 149 150 // When the video is decreased, set the waiting time for CPU adaptation to 151 // decrease video again. 152 void set_cpu_adapt_wait_time(uint32 cpu_adapt_wait_time) { 153 if (cpu_adapt_wait_time_ != static_cast<int>(cpu_adapt_wait_time)) { 154 LOG(LS_INFO) << "VAdapt Change Cpu Adapt Wait Time from: " 155 << cpu_adapt_wait_time_ << " to " 156 << cpu_adapt_wait_time; 157 cpu_adapt_wait_time_ = static_cast<int>(cpu_adapt_wait_time); 158 } 159 } 160 // CPU system load high threshold for reducing resolution. e.g. 0.85f 161 void set_high_system_threshold(float high_system_threshold) { 162 ASSERT(high_system_threshold <= 1.0f); 163 ASSERT(high_system_threshold >= 0.0f); 164 if (high_system_threshold_ != high_system_threshold) { 165 LOG(LS_INFO) << "VAdapt Change High System Threshold from: " 166 << high_system_threshold_ << " to " << high_system_threshold; 167 high_system_threshold_ = high_system_threshold; 168 } 169 } 170 float high_system_threshold() const { return high_system_threshold_; } 171 // CPU system load low threshold for increasing resolution. e.g. 0.70f 172 void set_low_system_threshold(float low_system_threshold) { 173 ASSERT(low_system_threshold <= 1.0f); 174 ASSERT(low_system_threshold >= 0.0f); 175 if (low_system_threshold_ != low_system_threshold) { 176 LOG(LS_INFO) << "VAdapt Change Low System Threshold from: " 177 << low_system_threshold_ << " to " << low_system_threshold; 178 low_system_threshold_ = low_system_threshold; 179 } 180 } 181 float low_system_threshold() const { return low_system_threshold_; } 182 // CPU process load threshold for reducing resolution. e.g. 0.10f 183 void set_process_threshold(float process_threshold) { 184 ASSERT(process_threshold <= 1.0f); 185 ASSERT(process_threshold >= 0.0f); 186 if (process_threshold_ != process_threshold) { 187 LOG(LS_INFO) << "VAdapt Change High Process Threshold from: " 188 << process_threshold_ << " to " << process_threshold; 189 process_threshold_ = process_threshold; 190 } 191 } 192 float process_threshold() const { return process_threshold_; } 193 194 // Handle the format request from the server via Jingle update message. 195 void OnOutputFormatRequest(const VideoFormat& format); 196 // Handle the resolution request from the encoder due to bandwidth changes. 197 void OnEncoderResolutionRequest(int width, int height, AdaptRequest request); 198 // Handle the resolution request for CPU overuse. 199 void OnCpuResolutionRequest(AdaptRequest request); 200 // Handle the CPU load provided by a CPU monitor. 201 void OnCpuLoadUpdated(int current_cpus, int max_cpus, 202 float process_load, float system_load); 203 204 sigslot::signal0<> SignalCpuAdaptationUnable; 205 206 private: 207 // Adapt to the minimum of the formats the server requests, the CPU wants, and 208 // the encoder wants. Returns true if resolution changed. 209 bool AdaptToMinimumFormat(int* new_width, int* new_height); 210 bool IsMinimumFormat(int pixels); 211 void StepPixelCount(CoordinatedVideoAdapter::AdaptRequest request, 212 int* num_pixels); 213 CoordinatedVideoAdapter::AdaptRequest FindCpuRequest( 214 int current_cpus, int max_cpus, 215 float process_load, float system_load); 216 217 bool cpu_adaptation_; // True if cpu adaptation is enabled. 218 bool cpu_smoothing_; // True if cpu smoothing is enabled (with adaptation). 219 bool gd_adaptation_; // True if gd adaptation is enabled. 220 bool view_adaptation_; // True if view adaptation is enabled. 221 bool view_switch_; // True if view switch is enabled. 222 int cpu_downgrade_count_; 223 int cpu_adapt_wait_time_; 224 // cpu system load thresholds relative to max cpus. 225 float high_system_threshold_; 226 float low_system_threshold_; 227 // cpu process load thresholds relative to current cpus. 228 float process_threshold_; 229 // Video formats that the server view requests, the CPU wants, and the encoder 230 // wants respectively. The adapted output format is the minimum of these. 231 int view_desired_num_pixels_; 232 int64 view_desired_interval_; 233 int encoder_desired_num_pixels_; 234 int cpu_desired_num_pixels_; 235 CoordinatedVideoAdapter::AdaptReason adapt_reason_; 236 // The critical section to protect handling requests. 237 talk_base::CriticalSection request_critical_section_; 238 239 // The weighted average of cpu load over time. It's always updated (if cpu 240 // adaptation is on), but only used if cpu_smoothing_ is set. 241 float system_load_average_; 242 243 DISALLOW_COPY_AND_ASSIGN(CoordinatedVideoAdapter); 244 }; 245 246 } // namespace cricket 247 248 #endif // TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT 249
