1 // libjingle 2 // Copyright 2010 Google Inc. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are met: 6 // 7 // 1. Redistributions of source code must retain the above copyright notice, 8 // this list of conditions and the following disclaimer. 9 // 2. Redistributions in binary form must reproduce the above copyright notice, 10 // this list of conditions and the following disclaimer in the documentation 11 // and/or other materials provided with the distribution. 12 // 3. The name of the author may not be used to endorse or promote products 13 // derived from this software without specific prior written permission. 14 // 15 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 16 // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 17 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 18 // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 19 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 24 // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 26 #include "talk/media/base/videoadapter.h" 27 28 #include <limits.h> // For INT_MAX 29 30 #include "talk/media/base/constants.h" 31 #include "talk/media/base/videocommon.h" 32 #include "talk/media/base/videoframe.h" 33 #include "webrtc/base/logging.h" 34 #include "webrtc/base/timeutils.h" 35 36 namespace cricket { 37 38 // TODO(fbarchard): Make downgrades settable 39 static const int kMaxCpuDowngrades = 2; // Downgrade at most 2 times for CPU. 40 // The number of cpu samples to require before adapting. This value depends on 41 // the cpu monitor sampling frequency being 2000ms. 42 static const int kCpuLoadMinSamples = 3; 43 // The amount of weight to give to each new cpu load sample. The lower the 44 // value, the slower we'll adapt to changing cpu conditions. 45 static const float kCpuLoadWeightCoefficient = 0.4f; 46 // The seed value for the cpu load moving average. 47 static const float kCpuLoadInitialAverage = 0.5f; 48 49 // Desktop needs 1/8 scale for HD (1280 x 720) to QQVGA (160 x 90) 50 static const float kScaleFactors[] = { 51 1.f / 1.f, // Full size. 52 3.f / 4.f, // 3/4 scale. 53 1.f / 2.f, // 1/2 scale. 54 3.f / 8.f, // 3/8 scale. 55 1.f / 4.f, // 1/4 scale. 56 3.f / 16.f, // 3/16 scale. 57 1.f / 8.f, // 1/8 scale. 58 0.f // End of table. 59 }; 60 61 // TODO(fbarchard): Use this table (optionally) for CPU and GD as well. 62 static const float kViewScaleFactors[] = { 63 1.f / 1.f, // Full size. 64 3.f / 4.f, // 3/4 scale. 65 2.f / 3.f, // 2/3 scale. // Allow 1080p to 720p. 66 1.f / 2.f, // 1/2 scale. 67 3.f / 8.f, // 3/8 scale. 68 1.f / 3.f, // 1/3 scale. // Allow 1080p to 360p. 69 1.f / 4.f, // 1/4 scale. 70 3.f / 16.f, // 3/16 scale. 71 1.f / 8.f, // 1/8 scale. 72 0.f // End of table. 73 }; 74 75 const float* VideoAdapter::GetViewScaleFactors() const { 76 return scale_third_ ? kViewScaleFactors : kScaleFactors; 77 } 78 79 // For resolutions that would scale down a little instead of up a little, 80 // bias toward scaling up a little. This will tend to choose 3/4 scale instead 81 // of 2/3 scale, when the 2/3 is not an exact match. 82 static const float kUpBias = -0.9f; 83 // Find the scale factor that, when applied to width and height, is closest 84 // to num_pixels. 85 float VideoAdapter::FindScale(const float* scale_factors, 86 const float upbias, 87 int width, int height, 88 int target_num_pixels) { 89 const float kMinNumPixels = 160 * 90; 90 if (!target_num_pixels) { 91 return 0.f; 92 } 93 float best_distance = static_cast<float>(INT_MAX); 94 float best_scale = 1.f; // Default to unscaled if nothing matches. 95 float pixels = static_cast<float>(width * height); 96 for (int i = 0; ; ++i) { 97 float scale = scale_factors[i]; 98 float test_num_pixels = pixels * scale * scale; 99 // Do not consider scale factors that produce too small images. 100 // Scale factor of 0 at end of table will also exit here. 101 if (test_num_pixels < kMinNumPixels) { 102 break; 103 } 104 float diff = target_num_pixels - test_num_pixels; 105 // If resolution is higher than desired, bias the difference based on 106 // preference for slightly larger for nearest, or avoid completely if 107 // looking for lower resolutions only. 108 if (diff < 0) { 109 diff = diff * kUpBias; 110 } 111 if (diff < best_distance) { 112 best_distance = diff; 113 best_scale = scale; 114 if (best_distance == 0) { // Found exact match. 115 break; 116 } 117 } 118 } 119 return best_scale; 120 } 121 122 // Find the closest scale factor. 123 float VideoAdapter::FindClosestScale(int width, int height, 124 int target_num_pixels) { 125 return FindScale(kScaleFactors, kUpBias, 126 width, height, target_num_pixels); 127 } 128 129 // Find the closest view scale factor. 130 float VideoAdapter::FindClosestViewScale(int width, int height, 131 int target_num_pixels) { 132 return FindScale(GetViewScaleFactors(), kUpBias, 133 width, height, target_num_pixels); 134 } 135 136 // Finds the scale factor that, when applied to width and height, produces 137 // fewer than num_pixels. 138 static const float kUpAvoidBias = -1000000000.f; 139 float VideoAdapter::FindLowerScale(int width, int height, 140 int target_num_pixels) { 141 return FindScale(GetViewScaleFactors(), kUpAvoidBias, 142 width, height, target_num_pixels); 143 } 144 145 // There are several frame sizes used by Adapter. This explains them 146 // input_format - set once by server to frame size expected from the camera. 147 // output_format - size that output would like to be. Includes framerate. 148 // output_num_pixels - size that output should be constrained to. Used to 149 // compute output_format from in_frame. 150 // in_frame - actual camera captured frame size, which is typically the same 151 // as input_format. This can also be rotated or cropped for aspect ratio. 152 // out_frame - actual frame output by adapter. Should be a direct scale of 153 // in_frame maintaining rotation and aspect ratio. 154 // OnOutputFormatRequest - server requests you send this resolution based on 155 // view requests. 156 // OnEncoderResolutionRequest - encoder requests you send this resolution based 157 // on bandwidth 158 // OnCpuLoadUpdated - cpu monitor requests you send this resolution based on 159 // cpu load. 160 161 /////////////////////////////////////////////////////////////////////// 162 // Implementation of VideoAdapter 163 VideoAdapter::VideoAdapter() 164 : output_num_pixels_(INT_MAX), 165 scale_third_(false), 166 frames_in_(0), 167 frames_out_(0), 168 frames_scaled_(0), 169 adaption_changes_(0), 170 previous_width_(0), 171 previous_height_(0), 172 black_output_(false), 173 is_black_(false), 174 interval_next_frame_(0) { 175 } 176 177 VideoAdapter::~VideoAdapter() { 178 } 179 180 void VideoAdapter::SetInputFormat(const VideoFormat& format) { 181 rtc::CritScope cs(&critical_section_); 182 int64 old_input_interval = input_format_.interval; 183 input_format_ = format; 184 output_format_.interval = rtc::_max( 185 output_format_.interval, input_format_.interval); 186 if (old_input_interval != input_format_.interval) { 187 LOG(LS_INFO) << "VAdapt input interval changed from " 188 << old_input_interval << " to " << input_format_.interval; 189 } 190 } 191 192 void CoordinatedVideoAdapter::SetInputFormat(const VideoFormat& format) { 193 int previous_width = input_format().width; 194 int previous_height = input_format().height; 195 bool is_resolution_change = previous_width > 0 && format.width > 0 && 196 (previous_width != format.width || 197 previous_height != format.height); 198 VideoAdapter::SetInputFormat(format); 199 if (is_resolution_change) { 200 int width, height; 201 // Trigger the adaptation logic again, to potentially reset the adaptation 202 // state for things like view requests that may not longer be capping 203 // output (or may now cap output). 204 AdaptToMinimumFormat(&width, &height); 205 LOG(LS_INFO) << "VAdapt Input Resolution Change: " 206 << "Previous input resolution: " 207 << previous_width << "x" << previous_height 208 << " New input resolution: " 209 << format.width << "x" << format.height 210 << " New output resolution: " 211 << width << "x" << height; 212 } 213 } 214 215 void CoordinatedVideoAdapter::set_cpu_smoothing(bool enable) { 216 LOG(LS_INFO) << "CPU smoothing is now " 217 << (enable ? "enabled" : "disabled"); 218 cpu_smoothing_ = enable; 219 } 220 221 void VideoAdapter::SetOutputFormat(const VideoFormat& format) { 222 rtc::CritScope cs(&critical_section_); 223 int64 old_output_interval = output_format_.interval; 224 output_format_ = format; 225 output_num_pixels_ = output_format_.width * output_format_.height; 226 output_format_.interval = rtc::_max( 227 output_format_.interval, input_format_.interval); 228 if (old_output_interval != output_format_.interval) { 229 LOG(LS_INFO) << "VAdapt output interval changed from " 230 << old_output_interval << " to " << output_format_.interval; 231 } 232 } 233 234 const VideoFormat& VideoAdapter::input_format() { 235 rtc::CritScope cs(&critical_section_); 236 return input_format_; 237 } 238 239 bool VideoAdapter::drops_all_frames() const { 240 return output_num_pixels_ == 0; 241 } 242 243 const VideoFormat& VideoAdapter::output_format() { 244 rtc::CritScope cs(&critical_section_); 245 return output_format_; 246 } 247 248 void VideoAdapter::SetBlackOutput(bool black) { 249 rtc::CritScope cs(&critical_section_); 250 black_output_ = black; 251 } 252 253 // Constrain output resolution to this many pixels overall 254 void VideoAdapter::SetOutputNumPixels(int num_pixels) { 255 output_num_pixels_ = num_pixels; 256 } 257 258 int VideoAdapter::GetOutputNumPixels() const { 259 return output_num_pixels_; 260 } 261 262 // TODO(fbarchard): Add AdaptFrameRate function that only drops frames but 263 // not resolution. 264 bool VideoAdapter::AdaptFrame(VideoFrame* in_frame, 265 VideoFrame** out_frame) { 266 rtc::CritScope cs(&critical_section_); 267 if (!in_frame || !out_frame) { 268 return false; 269 } 270 ++frames_in_; 271 272 // Update input to actual frame dimensions. 273 VideoFormat format(static_cast<int>(in_frame->GetWidth()), 274 static_cast<int>(in_frame->GetHeight()), 275 input_format_.interval, input_format_.fourcc); 276 SetInputFormat(format); 277 278 // Drop the input frame if necessary. 279 bool should_drop = false; 280 if (!output_num_pixels_) { 281 // Drop all frames as the output format is 0x0. 282 should_drop = true; 283 } else { 284 // Drop some frames based on input fps and output fps. 285 // Normally output fps is less than input fps. 286 // TODO(fbarchard): Consider adjusting interval to reflect the adjusted 287 // interval between frames after dropping some frames. 288 interval_next_frame_ += input_format_.interval; 289 if (output_format_.interval > 0) { 290 if (interval_next_frame_ >= output_format_.interval) { 291 interval_next_frame_ %= output_format_.interval; 292 } else { 293 should_drop = true; 294 } 295 } 296 } 297 if (should_drop) { 298 // Show VAdapt log every 90 frames dropped. (3 seconds) 299 if ((frames_in_ - frames_out_) % 90 == 0) { 300 // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed 301 // in default calls. 302 LOG(LS_INFO) << "VAdapt Drop Frame: scaled " << frames_scaled_ 303 << " / out " << frames_out_ 304 << " / in " << frames_in_ 305 << " Changes: " << adaption_changes_ 306 << " Input: " << in_frame->GetWidth() 307 << "x" << in_frame->GetHeight() 308 << " i" << input_format_.interval 309 << " Output: i" << output_format_.interval; 310 } 311 *out_frame = NULL; 312 return true; 313 } 314 315 float scale = 1.f; 316 if (output_num_pixels_ < input_format_.width * input_format_.height) { 317 scale = VideoAdapter::FindClosestViewScale( 318 static_cast<int>(in_frame->GetWidth()), 319 static_cast<int>(in_frame->GetHeight()), 320 output_num_pixels_); 321 output_format_.width = static_cast<int>(in_frame->GetWidth() * scale + .5f); 322 output_format_.height = static_cast<int>(in_frame->GetHeight() * scale + 323 .5f); 324 } else { 325 output_format_.width = static_cast<int>(in_frame->GetWidth()); 326 output_format_.height = static_cast<int>(in_frame->GetHeight()); 327 } 328 329 if (!black_output_ && 330 in_frame->GetWidth() == static_cast<size_t>(output_format_.width) && 331 in_frame->GetHeight() == static_cast<size_t>(output_format_.height)) { 332 // The dimensions are correct and we aren't muting, so use the input frame. 333 *out_frame = in_frame; 334 } else { 335 if (!StretchToOutputFrame(in_frame)) { 336 LOG(LS_VERBOSE) << "VAdapt Stretch Failed."; 337 return false; 338 } 339 340 *out_frame = output_frame_.get(); 341 } 342 343 ++frames_out_; 344 if (in_frame->GetWidth() != (*out_frame)->GetWidth() || 345 in_frame->GetHeight() != (*out_frame)->GetHeight()) { 346 ++frames_scaled_; 347 } 348 // Show VAdapt log every 90 frames output. (3 seconds) 349 // TODO(fbarchard): Consider GetLogSeverity() to change interval to less 350 // for LS_VERBOSE and more for LS_INFO. 351 bool show = (frames_out_) % 90 == 0; 352 353 // TODO(fbarchard): LOG the previous output resolution and track input 354 // resolution changes as well. Consider dropping the statistics into their 355 // own class which could be queried publically. 356 bool changed = false; 357 if (previous_width_ && (previous_width_ != (*out_frame)->GetWidth() || 358 previous_height_ != (*out_frame)->GetHeight())) { 359 show = true; 360 ++adaption_changes_; 361 changed = true; 362 } 363 if (show) { 364 // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed 365 // in default calls. 366 LOG(LS_INFO) << "VAdapt Frame: scaled " << frames_scaled_ 367 << " / out " << frames_out_ 368 << " / in " << frames_in_ 369 << " Changes: " << adaption_changes_ 370 << " Input: " << in_frame->GetWidth() 371 << "x" << in_frame->GetHeight() 372 << " i" << input_format_.interval 373 << " Scale: " << scale 374 << " Output: " << (*out_frame)->GetWidth() 375 << "x" << (*out_frame)->GetHeight() 376 << " i" << output_format_.interval 377 << " Changed: " << (changed ? "true" : "false"); 378 } 379 previous_width_ = (*out_frame)->GetWidth(); 380 previous_height_ = (*out_frame)->GetHeight(); 381 382 return true; 383 } 384 385 void VideoAdapter::set_scale_third(bool enable) { 386 LOG(LS_INFO) << "Video Adapter third scaling is now " 387 << (enable ? "enabled" : "disabled"); 388 scale_third_ = enable; 389 } 390 391 // Scale or Blacken the frame. Returns true if successful. 392 bool VideoAdapter::StretchToOutputFrame(const VideoFrame* in_frame) { 393 int output_width = output_format_.width; 394 int output_height = output_format_.height; 395 396 // Create and stretch the output frame if it has not been created yet or its 397 // size is not same as the expected. 398 bool stretched = false; 399 if (!output_frame_ || 400 output_frame_->GetWidth() != static_cast<size_t>(output_width) || 401 output_frame_->GetHeight() != static_cast<size_t>(output_height)) { 402 output_frame_.reset( 403 in_frame->Stretch(output_width, output_height, true, true)); 404 if (!output_frame_) { 405 LOG(LS_WARNING) << "Adapter failed to stretch frame to " 406 << output_width << "x" << output_height; 407 return false; 408 } 409 stretched = true; 410 is_black_ = false; 411 } 412 413 if (!black_output_) { 414 if (!stretched) { 415 // The output frame does not need to be blacken and has not been stretched 416 // from the input frame yet, stretch the input frame. This is the most 417 // common case. 418 in_frame->StretchToFrame(output_frame_.get(), true, true); 419 } 420 is_black_ = false; 421 } else { 422 if (!is_black_) { 423 output_frame_->SetToBlack(); 424 is_black_ = true; 425 } 426 output_frame_->SetElapsedTime(in_frame->GetElapsedTime()); 427 output_frame_->SetTimeStamp(in_frame->GetTimeStamp()); 428 } 429 430 return true; 431 } 432 433 /////////////////////////////////////////////////////////////////////// 434 // Implementation of CoordinatedVideoAdapter 435 CoordinatedVideoAdapter::CoordinatedVideoAdapter() 436 : cpu_adaptation_(true), 437 cpu_smoothing_(false), 438 gd_adaptation_(true), 439 view_adaptation_(true), 440 view_switch_(false), 441 cpu_downgrade_count_(0), 442 cpu_load_min_samples_(kCpuLoadMinSamples), 443 cpu_load_num_samples_(0), 444 high_system_threshold_(kHighSystemCpuThreshold), 445 low_system_threshold_(kLowSystemCpuThreshold), 446 process_threshold_(kProcessCpuThreshold), 447 view_desired_num_pixels_(INT_MAX), 448 view_desired_interval_(0), 449 encoder_desired_num_pixels_(INT_MAX), 450 cpu_desired_num_pixels_(INT_MAX), 451 adapt_reason_(ADAPTREASON_NONE), 452 system_load_average_(kCpuLoadInitialAverage) { 453 } 454 455 // Helper function to UPGRADE or DOWNGRADE a number of pixels 456 void CoordinatedVideoAdapter::StepPixelCount( 457 CoordinatedVideoAdapter::AdaptRequest request, 458 int* num_pixels) { 459 switch (request) { 460 case CoordinatedVideoAdapter::DOWNGRADE: 461 *num_pixels /= 2; 462 break; 463 464 case CoordinatedVideoAdapter::UPGRADE: 465 *num_pixels *= 2; 466 break; 467 468 default: // No change in pixel count 469 break; 470 } 471 return; 472 } 473 474 // Find the adaptation request of the cpu based on the load. Return UPGRADE if 475 // the load is low, DOWNGRADE if the load is high, and KEEP otherwise. 476 CoordinatedVideoAdapter::AdaptRequest CoordinatedVideoAdapter::FindCpuRequest( 477 int current_cpus, int max_cpus, 478 float process_load, float system_load) { 479 // Downgrade if system is high and plugin is at least more than midrange. 480 if (system_load >= high_system_threshold_ * max_cpus && 481 process_load >= process_threshold_ * current_cpus) { 482 return CoordinatedVideoAdapter::DOWNGRADE; 483 // Upgrade if system is low. 484 } else if (system_load < low_system_threshold_ * max_cpus) { 485 return CoordinatedVideoAdapter::UPGRADE; 486 } 487 return CoordinatedVideoAdapter::KEEP; 488 } 489 490 // A remote view request for a new resolution. 491 void CoordinatedVideoAdapter::OnOutputFormatRequest(const VideoFormat& format) { 492 rtc::CritScope cs(&request_critical_section_); 493 if (!view_adaptation_) { 494 return; 495 } 496 // Set output for initial aspect ratio in mediachannel unittests. 497 int old_num_pixels = GetOutputNumPixels(); 498 SetOutputFormat(format); 499 SetOutputNumPixels(old_num_pixels); 500 view_desired_num_pixels_ = format.width * format.height; 501 view_desired_interval_ = format.interval; 502 int new_width, new_height; 503 bool changed = AdaptToMinimumFormat(&new_width, &new_height); 504 LOG(LS_INFO) << "VAdapt View Request: " 505 << format.width << "x" << format.height 506 << " Pixels: " << view_desired_num_pixels_ 507 << " Changed: " << (changed ? "true" : "false") 508 << " To: " << new_width << "x" << new_height; 509 } 510 511 void CoordinatedVideoAdapter::set_cpu_load_min_samples( 512 int cpu_load_min_samples) { 513 if (cpu_load_min_samples_ != cpu_load_min_samples) { 514 LOG(LS_INFO) << "VAdapt Change Cpu Adapt Min Samples from: " 515 << cpu_load_min_samples_ << " to " 516 << cpu_load_min_samples; 517 cpu_load_min_samples_ = cpu_load_min_samples; 518 } 519 } 520 521 void CoordinatedVideoAdapter::set_high_system_threshold( 522 float high_system_threshold) { 523 ASSERT(high_system_threshold <= 1.0f); 524 ASSERT(high_system_threshold >= 0.0f); 525 if (high_system_threshold_ != high_system_threshold) { 526 LOG(LS_INFO) << "VAdapt Change High System Threshold from: " 527 << high_system_threshold_ << " to " << high_system_threshold; 528 high_system_threshold_ = high_system_threshold; 529 } 530 } 531 532 void CoordinatedVideoAdapter::set_low_system_threshold( 533 float low_system_threshold) { 534 ASSERT(low_system_threshold <= 1.0f); 535 ASSERT(low_system_threshold >= 0.0f); 536 if (low_system_threshold_ != low_system_threshold) { 537 LOG(LS_INFO) << "VAdapt Change Low System Threshold from: " 538 << low_system_threshold_ << " to " << low_system_threshold; 539 low_system_threshold_ = low_system_threshold; 540 } 541 } 542 543 void CoordinatedVideoAdapter::set_process_threshold(float process_threshold) { 544 ASSERT(process_threshold <= 1.0f); 545 ASSERT(process_threshold >= 0.0f); 546 if (process_threshold_ != process_threshold) { 547 LOG(LS_INFO) << "VAdapt Change High Process Threshold from: " 548 << process_threshold_ << " to " << process_threshold; 549 process_threshold_ = process_threshold; 550 } 551 } 552 553 // A Bandwidth GD request for new resolution 554 void CoordinatedVideoAdapter::OnEncoderResolutionRequest( 555 int width, int height, AdaptRequest request) { 556 rtc::CritScope cs(&request_critical_section_); 557 if (!gd_adaptation_) { 558 return; 559 } 560 int old_encoder_desired_num_pixels = encoder_desired_num_pixels_; 561 if (KEEP != request) { 562 int new_encoder_desired_num_pixels = width * height; 563 int old_num_pixels = GetOutputNumPixels(); 564 if (new_encoder_desired_num_pixels != old_num_pixels) { 565 LOG(LS_VERBOSE) << "VAdapt GD resolution stale. Ignored"; 566 } else { 567 // Update the encoder desired format based on the request. 568 encoder_desired_num_pixels_ = new_encoder_desired_num_pixels; 569 StepPixelCount(request, &encoder_desired_num_pixels_); 570 } 571 } 572 int new_width, new_height; 573 bool changed = AdaptToMinimumFormat(&new_width, &new_height); 574 575 // Ignore up or keep if no change. 576 if (DOWNGRADE != request && view_switch_ && !changed) { 577 encoder_desired_num_pixels_ = old_encoder_desired_num_pixels; 578 LOG(LS_VERBOSE) << "VAdapt ignoring GD request."; 579 } 580 581 LOG(LS_INFO) << "VAdapt GD Request: " 582 << (DOWNGRADE == request ? "down" : 583 (UPGRADE == request ? "up" : "keep")) 584 << " From: " << width << "x" << height 585 << " Pixels: " << encoder_desired_num_pixels_ 586 << " Changed: " << (changed ? "true" : "false") 587 << " To: " << new_width << "x" << new_height; 588 } 589 590 // A Bandwidth GD request for new resolution 591 void CoordinatedVideoAdapter::OnCpuResolutionRequest(AdaptRequest request) { 592 rtc::CritScope cs(&request_critical_section_); 593 if (!cpu_adaptation_) { 594 return; 595 } 596 // Update how many times we have downgraded due to the cpu load. 597 switch (request) { 598 case DOWNGRADE: 599 // Ignore downgrades if we have downgraded the maximum times. 600 if (cpu_downgrade_count_ < kMaxCpuDowngrades) { 601 ++cpu_downgrade_count_; 602 } else { 603 LOG(LS_VERBOSE) << "VAdapt CPU load high but do not downgrade " 604 "because maximum downgrades reached"; 605 SignalCpuAdaptationUnable(); 606 } 607 break; 608 case UPGRADE: 609 if (cpu_downgrade_count_ > 0) { 610 bool is_min = IsMinimumFormat(cpu_desired_num_pixels_); 611 if (is_min) { 612 --cpu_downgrade_count_; 613 } else { 614 LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade " 615 "because cpu is not limiting resolution"; 616 } 617 } else { 618 LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade " 619 "because minimum downgrades reached"; 620 } 621 break; 622 case KEEP: 623 default: 624 break; 625 } 626 if (KEEP != request) { 627 // TODO(fbarchard): compute stepping up/down from OutputNumPixels but 628 // clamp to inputpixels / 4 (2 steps) 629 cpu_desired_num_pixels_ = cpu_downgrade_count_ == 0 ? INT_MAX : 630 static_cast<int>(input_format().width * input_format().height >> 631 cpu_downgrade_count_); 632 } 633 int new_width, new_height; 634 bool changed = AdaptToMinimumFormat(&new_width, &new_height); 635 LOG(LS_INFO) << "VAdapt CPU Request: " 636 << (DOWNGRADE == request ? "down" : 637 (UPGRADE == request ? "up" : "keep")) 638 << " Steps: " << cpu_downgrade_count_ 639 << " Changed: " << (changed ? "true" : "false") 640 << " To: " << new_width << "x" << new_height; 641 } 642 643 // A CPU request for new resolution 644 // TODO(fbarchard): Move outside adapter. 645 void CoordinatedVideoAdapter::OnCpuLoadUpdated( 646 int current_cpus, int max_cpus, float process_load, float system_load) { 647 rtc::CritScope cs(&request_critical_section_); 648 if (!cpu_adaptation_) { 649 return; 650 } 651 // Update the moving average of system load. Even if we aren't smoothing, 652 // we'll still calculate this information, in case smoothing is later enabled. 653 system_load_average_ = kCpuLoadWeightCoefficient * system_load + 654 (1.0f - kCpuLoadWeightCoefficient) * system_load_average_; 655 ++cpu_load_num_samples_; 656 if (cpu_smoothing_) { 657 system_load = system_load_average_; 658 } 659 AdaptRequest request = FindCpuRequest(current_cpus, max_cpus, 660 process_load, system_load); 661 // Make sure we're not adapting too quickly. 662 if (request != KEEP) { 663 if (cpu_load_num_samples_ < cpu_load_min_samples_) { 664 LOG(LS_VERBOSE) << "VAdapt CPU load high/low but do not adapt until " 665 << (cpu_load_min_samples_ - cpu_load_num_samples_) 666 << " more samples"; 667 request = KEEP; 668 } 669 } 670 671 OnCpuResolutionRequest(request); 672 } 673 674 // Called by cpu adapter on up requests. 675 bool CoordinatedVideoAdapter::IsMinimumFormat(int pixels) { 676 // Find closest scale factor that matches input resolution to min_num_pixels 677 // and set that for output resolution. This is not needed for VideoAdapter, 678 // but provides feedback to unittests and users on expected resolution. 679 // Actual resolution is based on input frame. 680 VideoFormat new_output = output_format(); 681 VideoFormat input = input_format(); 682 if (input_format().IsSize0x0()) { 683 input = new_output; 684 } 685 float scale = 1.0f; 686 if (!input.IsSize0x0()) { 687 scale = FindClosestScale(input.width, 688 input.height, 689 pixels); 690 } 691 new_output.width = static_cast<int>(input.width * scale + .5f); 692 new_output.height = static_cast<int>(input.height * scale + .5f); 693 int new_pixels = new_output.width * new_output.height; 694 int num_pixels = GetOutputNumPixels(); 695 return new_pixels <= num_pixels; 696 } 697 698 // Called by all coordinators when there is a change. 699 bool CoordinatedVideoAdapter::AdaptToMinimumFormat(int* new_width, 700 int* new_height) { 701 VideoFormat new_output = output_format(); 702 VideoFormat input = input_format(); 703 if (input_format().IsSize0x0()) { 704 input = new_output; 705 } 706 int old_num_pixels = GetOutputNumPixels(); 707 int min_num_pixels = INT_MAX; 708 adapt_reason_ = ADAPTREASON_NONE; 709 710 // Reduce resolution based on encoder bandwidth (GD). 711 if (encoder_desired_num_pixels_ && 712 (encoder_desired_num_pixels_ < min_num_pixels)) { 713 adapt_reason_ |= ADAPTREASON_BANDWIDTH; 714 min_num_pixels = encoder_desired_num_pixels_; 715 } 716 // Reduce resolution based on CPU. 717 if (cpu_adaptation_ && cpu_desired_num_pixels_ && 718 (cpu_desired_num_pixels_ <= min_num_pixels)) { 719 if (cpu_desired_num_pixels_ < min_num_pixels) { 720 adapt_reason_ = ADAPTREASON_CPU; 721 } else { 722 adapt_reason_ |= ADAPTREASON_CPU; 723 } 724 min_num_pixels = cpu_desired_num_pixels_; 725 } 726 // Round resolution for GD or CPU to allow 1/2 to map to 9/16. 727 if (!input.IsSize0x0() && min_num_pixels != INT_MAX) { 728 float scale = FindClosestScale(input.width, input.height, min_num_pixels); 729 min_num_pixels = static_cast<int>(input.width * scale + .5f) * 730 static_cast<int>(input.height * scale + .5f); 731 } 732 // Reduce resolution based on View Request. 733 if (view_desired_num_pixels_ <= min_num_pixels) { 734 if (view_desired_num_pixels_ < min_num_pixels) { 735 adapt_reason_ = ADAPTREASON_VIEW; 736 } else { 737 adapt_reason_ |= ADAPTREASON_VIEW; 738 } 739 min_num_pixels = view_desired_num_pixels_; 740 } 741 // Snap to a scale factor. 742 float scale = 1.0f; 743 if (!input.IsSize0x0()) { 744 scale = FindLowerScale(input.width, input.height, min_num_pixels); 745 min_num_pixels = static_cast<int>(input.width * scale + .5f) * 746 static_cast<int>(input.height * scale + .5f); 747 } 748 if (scale == 1.0f) { 749 adapt_reason_ = ADAPTREASON_NONE; 750 } 751 *new_width = new_output.width = static_cast<int>(input.width * scale + .5f); 752 *new_height = new_output.height = static_cast<int>(input.height * scale + 753 .5f); 754 SetOutputNumPixels(min_num_pixels); 755 756 new_output.interval = view_desired_interval_; 757 SetOutputFormat(new_output); 758 int new_num_pixels = GetOutputNumPixels(); 759 bool changed = new_num_pixels != old_num_pixels; 760 761 static const char* kReasons[8] = { 762 "None", 763 "CPU", 764 "BANDWIDTH", 765 "CPU+BANDWIDTH", 766 "VIEW", 767 "CPU+VIEW", 768 "BANDWIDTH+VIEW", 769 "CPU+BANDWIDTH+VIEW", 770 }; 771 772 LOG(LS_VERBOSE) << "VAdapt Status View: " << view_desired_num_pixels_ 773 << " GD: " << encoder_desired_num_pixels_ 774 << " CPU: " << cpu_desired_num_pixels_ 775 << " Pixels: " << min_num_pixels 776 << " Input: " << input.width 777 << "x" << input.height 778 << " Scale: " << scale 779 << " Resolution: " << new_output.width 780 << "x" << new_output.height 781 << " Changed: " << (changed ? "true" : "false") 782 << " Reason: " << kReasons[adapt_reason_]; 783 784 if (changed) { 785 // When any adaptation occurs, historic CPU load levels are no longer 786 // accurate. Clear out our state so we can re-learn at the new normal. 787 cpu_load_num_samples_ = 0; 788 system_load_average_ = kCpuLoadInitialAverage; 789 } 790 791 return changed; 792 } 793 794 } // namespace cricket 795
