1 // libjingle 2 // Copyright 2010 Google Inc. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are met: 6 // 7 // 1. Redistributions of source code must retain the above copyright notice, 8 // this list of conditions and the following disclaimer. 9 // 2. Redistributions in binary form must reproduce the above copyright notice, 10 // this list of conditions and the following disclaimer in the documentation 11 // and/or other materials provided with the distribution. 12 // 3. The name of the author may not be used to endorse or promote products 13 // derived from this software without specific prior written permission. 14 // 15 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 16 // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 17 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 18 // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 19 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 24 // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 // 26 // Implementation file of class VideoCapturer. 27 28 #include "talk/media/base/videocapturer.h" 29 30 #include <algorithm> 31 32 #if !defined(DISABLE_YUV) 33 #include "libyuv/scale_argb.h" 34 #endif 35 #include "talk/media/base/videoframefactory.h" 36 #include "talk/media/base/videoprocessor.h" 37 #include "webrtc/base/common.h" 38 #include "webrtc/base/logging.h" 39 #include "webrtc/base/systeminfo.h" 40 41 #if defined(HAVE_WEBRTC_VIDEO) 42 #include "talk/media/webrtc/webrtcvideoframe.h" 43 #include "talk/media/webrtc/webrtcvideoframefactory.h" 44 #endif // HAVE_WEBRTC_VIDEO 45 46 namespace cricket { 47 48 namespace { 49 50 // TODO(thorcarpenter): This is a BIG hack to flush the system with black 51 // frames. Frontends should coordinate to update the video state of a muted 52 // user. When all frontends to this consider removing the black frame business. 53 const int kNumBlackFramesOnMute = 30; 54 55 // MessageHandler constants. 56 enum { 57 MSG_DO_PAUSE = 0, 58 MSG_DO_UNPAUSE, 59 MSG_STATE_CHANGE 60 }; 61 62 static const int64 kMaxDistance = ~(static_cast<int64>(1) << 63); 63 #ifdef LINUX 64 static const int kYU12Penalty = 16; // Needs to be higher than MJPG index. 65 #endif 66 static const int kDefaultScreencastFps = 5; 67 typedef rtc::TypedMessageData<CaptureState> StateChangeParams; 68 69 // Limit stats data collections to ~20 seconds of 30fps data before dropping 70 // old data in case stats aren't reset for long periods of time. 71 static const size_t kMaxAccumulatorSize = 600; 72 73 } // namespace 74 75 ///////////////////////////////////////////////////////////////////// 76 // Implementation of struct CapturedFrame 77 ///////////////////////////////////////////////////////////////////// 78 CapturedFrame::CapturedFrame() 79 : width(0), 80 height(0), 81 fourcc(0), 82 pixel_width(0), 83 pixel_height(0), 84 elapsed_time(0), 85 time_stamp(0), 86 data_size(0), 87 rotation(0), 88 data(NULL) {} 89 90 // TODO(fbarchard): Remove this function once lmimediaengine stops using it. 91 bool CapturedFrame::GetDataSize(uint32* size) const { 92 if (!size || data_size == CapturedFrame::kUnknownDataSize) { 93 return false; 94 } 95 *size = data_size; 96 return true; 97 } 98 99 ///////////////////////////////////////////////////////////////////// 100 // Implementation of class VideoCapturer 101 ///////////////////////////////////////////////////////////////////// 102 VideoCapturer::VideoCapturer() 103 : thread_(rtc::Thread::Current()), 104 adapt_frame_drops_data_(kMaxAccumulatorSize), 105 effect_frame_drops_data_(kMaxAccumulatorSize), 106 frame_time_data_(kMaxAccumulatorSize) { 107 Construct(); 108 } 109 110 VideoCapturer::VideoCapturer(rtc::Thread* thread) 111 : thread_(thread), 112 adapt_frame_drops_data_(kMaxAccumulatorSize), 113 effect_frame_drops_data_(kMaxAccumulatorSize), 114 frame_time_data_(kMaxAccumulatorSize) { 115 Construct(); 116 } 117 118 void VideoCapturer::Construct() { 119 ClearAspectRatio(); 120 enable_camera_list_ = false; 121 square_pixel_aspect_ratio_ = false; 122 capture_state_ = CS_STOPPED; 123 SignalFrameCaptured.connect(this, &VideoCapturer::OnFrameCaptured); 124 scaled_width_ = 0; 125 scaled_height_ = 0; 126 screencast_max_pixels_ = 0; 127 muted_ = false; 128 black_frame_count_down_ = kNumBlackFramesOnMute; 129 enable_video_adapter_ = true; 130 adapt_frame_drops_ = 0; 131 effect_frame_drops_ = 0; 132 previous_frame_time_ = 0.0; 133 #ifdef HAVE_WEBRTC_VIDEO 134 // There are lots of video capturers out there that don't call 135 // set_frame_factory. We can either go change all of them, or we 136 // can set this default. 137 // TODO(pthatcher): Remove this hack and require the frame factory 138 // to be passed in the constructor. 139 set_frame_factory(new WebRtcVideoFrameFactory()); 140 #endif 141 } 142 143 const std::vector<VideoFormat>* VideoCapturer::GetSupportedFormats() const { 144 return &filtered_supported_formats_; 145 } 146 147 bool VideoCapturer::StartCapturing(const VideoFormat& capture_format) { 148 previous_frame_time_ = frame_length_time_reporter_.TimerNow(); 149 CaptureState result = Start(capture_format); 150 const bool success = (result == CS_RUNNING) || (result == CS_STARTING); 151 if (!success) { 152 return false; 153 } 154 if (result == CS_RUNNING) { 155 SetCaptureState(result); 156 } 157 return true; 158 } 159 160 void VideoCapturer::UpdateAspectRatio(int ratio_w, int ratio_h) { 161 if (ratio_w == 0 || ratio_h == 0) { 162 LOG(LS_WARNING) << "UpdateAspectRatio ignored invalid ratio: " 163 << ratio_w << "x" << ratio_h; 164 return; 165 } 166 ratio_w_ = ratio_w; 167 ratio_h_ = ratio_h; 168 } 169 170 void VideoCapturer::ClearAspectRatio() { 171 ratio_w_ = 0; 172 ratio_h_ = 0; 173 } 174 175 // Override this to have more control of how your device is started/stopped. 176 bool VideoCapturer::Pause(bool pause) { 177 if (pause) { 178 if (capture_state() == CS_PAUSED) { 179 return true; 180 } 181 bool is_running = capture_state() == CS_STARTING || 182 capture_state() == CS_RUNNING; 183 if (!is_running) { 184 LOG(LS_ERROR) << "Cannot pause a stopped camera."; 185 return false; 186 } 187 LOG(LS_INFO) << "Pausing a camera."; 188 rtc::scoped_ptr<VideoFormat> capture_format_when_paused( 189 capture_format_ ? new VideoFormat(*capture_format_) : NULL); 190 Stop(); 191 SetCaptureState(CS_PAUSED); 192 // If you override this function be sure to restore the capture format 193 // after calling Stop(). 194 SetCaptureFormat(capture_format_when_paused.get()); 195 } else { // Unpause. 196 if (capture_state() != CS_PAUSED) { 197 LOG(LS_WARNING) << "Cannot unpause a camera that hasn't been paused."; 198 return false; 199 } 200 if (!capture_format_) { 201 LOG(LS_ERROR) << "Missing capture_format_, cannot unpause a camera."; 202 return false; 203 } 204 if (muted_) { 205 LOG(LS_WARNING) << "Camera cannot be unpaused while muted."; 206 return false; 207 } 208 LOG(LS_INFO) << "Unpausing a camera."; 209 if (!Start(*capture_format_)) { 210 LOG(LS_ERROR) << "Camera failed to start when unpausing."; 211 return false; 212 } 213 } 214 return true; 215 } 216 217 bool VideoCapturer::Restart(const VideoFormat& capture_format) { 218 if (!IsRunning()) { 219 return StartCapturing(capture_format); 220 } 221 222 if (GetCaptureFormat() != NULL && *GetCaptureFormat() == capture_format) { 223 // The reqested format is the same; nothing to do. 224 return true; 225 } 226 227 Stop(); 228 return StartCapturing(capture_format); 229 } 230 231 bool VideoCapturer::MuteToBlackThenPause(bool muted) { 232 if (muted == IsMuted()) { 233 return true; 234 } 235 236 LOG(LS_INFO) << (muted ? "Muting" : "Unmuting") << " this video capturer."; 237 muted_ = muted; // Do this before calling Pause(). 238 if (muted) { 239 // Reset black frame count down. 240 black_frame_count_down_ = kNumBlackFramesOnMute; 241 // Following frames will be overritten with black, then the camera will be 242 // paused. 243 return true; 244 } 245 // Start the camera. 246 thread_->Clear(this, MSG_DO_PAUSE); 247 return Pause(false); 248 } 249 250 void VideoCapturer::SetSupportedFormats( 251 const std::vector<VideoFormat>& formats) { 252 supported_formats_ = formats; 253 UpdateFilteredSupportedFormats(); 254 } 255 256 bool VideoCapturer::GetBestCaptureFormat(const VideoFormat& format, 257 VideoFormat* best_format) { 258 // TODO(fbarchard): Directly support max_format. 259 UpdateFilteredSupportedFormats(); 260 const std::vector<VideoFormat>* supported_formats = GetSupportedFormats(); 261 262 if (supported_formats->empty()) { 263 return false; 264 } 265 LOG(LS_INFO) << " Capture Requested " << format.ToString(); 266 int64 best_distance = kMaxDistance; 267 std::vector<VideoFormat>::const_iterator best = supported_formats->end(); 268 std::vector<VideoFormat>::const_iterator i; 269 for (i = supported_formats->begin(); i != supported_formats->end(); ++i) { 270 int64 distance = GetFormatDistance(format, *i); 271 // TODO(fbarchard): Reduce to LS_VERBOSE if/when camera capture is 272 // relatively bug free. 273 LOG(LS_INFO) << " Supported " << i->ToString() << " distance " << distance; 274 if (distance < best_distance) { 275 best_distance = distance; 276 best = i; 277 } 278 } 279 if (supported_formats->end() == best) { 280 LOG(LS_ERROR) << " No acceptable camera format found"; 281 return false; 282 } 283 284 if (best_format) { 285 best_format->width = best->width; 286 best_format->height = best->height; 287 best_format->fourcc = best->fourcc; 288 best_format->interval = best->interval; 289 LOG(LS_INFO) << " Best " << best_format->ToString() << " Interval " 290 << best_format->interval << " distance " << best_distance; 291 } 292 return true; 293 } 294 295 void VideoCapturer::AddVideoProcessor(VideoProcessor* video_processor) { 296 rtc::CritScope cs(&crit_); 297 ASSERT(std::find(video_processors_.begin(), video_processors_.end(), 298 video_processor) == video_processors_.end()); 299 video_processors_.push_back(video_processor); 300 } 301 302 bool VideoCapturer::RemoveVideoProcessor(VideoProcessor* video_processor) { 303 rtc::CritScope cs(&crit_); 304 VideoProcessors::iterator found = std::find( 305 video_processors_.begin(), video_processors_.end(), video_processor); 306 if (found == video_processors_.end()) { 307 return false; 308 } 309 video_processors_.erase(found); 310 return true; 311 } 312 313 void VideoCapturer::ConstrainSupportedFormats(const VideoFormat& max_format) { 314 max_format_.reset(new VideoFormat(max_format)); 315 LOG(LS_VERBOSE) << " ConstrainSupportedFormats " << max_format.ToString(); 316 UpdateFilteredSupportedFormats(); 317 } 318 319 std::string VideoCapturer::ToString(const CapturedFrame* captured_frame) const { 320 std::string fourcc_name = GetFourccName(captured_frame->fourcc) + " "; 321 for (std::string::const_iterator i = fourcc_name.begin(); 322 i < fourcc_name.end(); ++i) { 323 // Test character is printable; Avoid isprint() which asserts on negatives. 324 if (*i < 32 || *i >= 127) { 325 fourcc_name = ""; 326 break; 327 } 328 } 329 330 std::ostringstream ss; 331 ss << fourcc_name << captured_frame->width << "x" << captured_frame->height 332 << "x" << VideoFormat::IntervalToFpsFloat(captured_frame->elapsed_time); 333 return ss.str(); 334 } 335 336 void VideoCapturer::GetStats(VariableInfo<int>* adapt_drops_stats, 337 VariableInfo<int>* effect_drops_stats, 338 VariableInfo<double>* frame_time_stats, 339 VideoFormat* last_captured_frame_format) { 340 rtc::CritScope cs(&frame_stats_crit_); 341 GetVariableSnapshot(adapt_frame_drops_data_, adapt_drops_stats); 342 GetVariableSnapshot(effect_frame_drops_data_, effect_drops_stats); 343 GetVariableSnapshot(frame_time_data_, frame_time_stats); 344 *last_captured_frame_format = last_captured_frame_format_; 345 346 adapt_frame_drops_data_.Reset(); 347 effect_frame_drops_data_.Reset(); 348 frame_time_data_.Reset(); 349 } 350 351 void VideoCapturer::OnFrameCaptured(VideoCapturer*, 352 const CapturedFrame* captured_frame) { 353 if (muted_) { 354 if (black_frame_count_down_ == 0) { 355 thread_->Post(this, MSG_DO_PAUSE, NULL); 356 } else { 357 --black_frame_count_down_; 358 } 359 } 360 361 if (SignalVideoFrame.is_empty()) { 362 return; 363 } 364 #if !defined(DISABLE_YUV) 365 if (IsScreencast()) { 366 int scaled_width, scaled_height; 367 if (screencast_max_pixels_ > 0) { 368 ComputeScaleMaxPixels(captured_frame->width, captured_frame->height, 369 screencast_max_pixels_, &scaled_width, &scaled_height); 370 } else { 371 int desired_screencast_fps = capture_format_.get() ? 372 VideoFormat::IntervalToFps(capture_format_->interval) : 373 kDefaultScreencastFps; 374 ComputeScale(captured_frame->width, captured_frame->height, 375 desired_screencast_fps, &scaled_width, &scaled_height); 376 } 377 378 if (FOURCC_ARGB == captured_frame->fourcc && 379 (scaled_width != captured_frame->width || 380 scaled_height != captured_frame->height)) { 381 if (scaled_width != scaled_width_ || scaled_height != scaled_height_) { 382 LOG(LS_INFO) << "Scaling Screencast from " 383 << captured_frame->width << "x" 384 << captured_frame->height << " to " 385 << scaled_width << "x" << scaled_height; 386 scaled_width_ = scaled_width; 387 scaled_height_ = scaled_height; 388 } 389 CapturedFrame* modified_frame = 390 const_cast<CapturedFrame*>(captured_frame); 391 // Compute new width such that width * height is less than maximum but 392 // maintains original captured frame aspect ratio. 393 // Round down width to multiple of 4 so odd width won't round up beyond 394 // maximum, and so chroma channel is even width to simplify spatial 395 // resampling. 396 libyuv::ARGBScale(reinterpret_cast<const uint8*>(captured_frame->data), 397 captured_frame->width * 4, captured_frame->width, 398 captured_frame->height, 399 reinterpret_cast<uint8*>(modified_frame->data), 400 scaled_width * 4, scaled_width, scaled_height, 401 libyuv::kFilterBilinear); 402 modified_frame->width = scaled_width; 403 modified_frame->height = scaled_height; 404 modified_frame->data_size = scaled_width * 4 * scaled_height; 405 } 406 } 407 408 const int kYuy2Bpp = 2; 409 const int kArgbBpp = 4; 410 // TODO(fbarchard): Make a helper function to adjust pixels to square. 411 // TODO(fbarchard): Hook up experiment to scaling. 412 // TODO(fbarchard): Avoid scale and convert if muted. 413 // Temporary buffer is scoped here so it will persist until i420_frame.Init() 414 // makes a copy of the frame, converting to I420. 415 rtc::scoped_ptr<uint8[]> temp_buffer; 416 // YUY2 can be scaled vertically using an ARGB scaler. Aspect ratio is only 417 // a problem on OSX. OSX always converts webcams to YUY2 or UYVY. 418 bool can_scale = 419 FOURCC_YUY2 == CanonicalFourCC(captured_frame->fourcc) || 420 FOURCC_UYVY == CanonicalFourCC(captured_frame->fourcc); 421 422 // If pixels are not square, optionally use vertical scaling to make them 423 // square. Square pixels simplify the rest of the pipeline, including 424 // effects and rendering. 425 if (can_scale && square_pixel_aspect_ratio_ && 426 captured_frame->pixel_width != captured_frame->pixel_height) { 427 int scaled_width, scaled_height; 428 // modified_frame points to the captured_frame but with const casted away 429 // so it can be modified. 430 CapturedFrame* modified_frame = const_cast<CapturedFrame*>(captured_frame); 431 // Compute the frame size that makes pixels square pixel aspect ratio. 432 ComputeScaleToSquarePixels(captured_frame->width, captured_frame->height, 433 captured_frame->pixel_width, 434 captured_frame->pixel_height, 435 &scaled_width, &scaled_height); 436 437 if (scaled_width != scaled_width_ || scaled_height != scaled_height_) { 438 LOG(LS_INFO) << "Scaling WebCam from " 439 << captured_frame->width << "x" 440 << captured_frame->height << " to " 441 << scaled_width << "x" << scaled_height 442 << " for PAR " 443 << captured_frame->pixel_width << "x" 444 << captured_frame->pixel_height; 445 scaled_width_ = scaled_width; 446 scaled_height_ = scaled_height; 447 } 448 const int modified_frame_size = scaled_width * scaled_height * kYuy2Bpp; 449 uint8* temp_buffer_data; 450 // Pixels are wide and short; Increasing height. Requires temporary buffer. 451 if (scaled_height > captured_frame->height) { 452 temp_buffer.reset(new uint8[modified_frame_size]); 453 temp_buffer_data = temp_buffer.get(); 454 } else { 455 // Pixels are narrow and tall; Decreasing height. Scale will be done 456 // in place. 457 temp_buffer_data = reinterpret_cast<uint8*>(captured_frame->data); 458 } 459 460 // Use ARGBScaler to vertically scale the YUY2 image, adjusting for 16 bpp. 461 libyuv::ARGBScale(reinterpret_cast<const uint8*>(captured_frame->data), 462 captured_frame->width * kYuy2Bpp, // Stride for YUY2. 463 captured_frame->width * kYuy2Bpp / kArgbBpp, // Width. 464 abs(captured_frame->height), // Height. 465 temp_buffer_data, 466 scaled_width * kYuy2Bpp, // Stride for YUY2. 467 scaled_width * kYuy2Bpp / kArgbBpp, // Width. 468 abs(scaled_height), // New height. 469 libyuv::kFilterBilinear); 470 modified_frame->width = scaled_width; 471 modified_frame->height = scaled_height; 472 modified_frame->pixel_width = 1; 473 modified_frame->pixel_height = 1; 474 modified_frame->data_size = modified_frame_size; 475 modified_frame->data = temp_buffer_data; 476 } 477 #endif // !DISABLE_YUV 478 479 // Size to crop captured frame to. This adjusts the captured frames 480 // aspect ratio to match the final view aspect ratio, considering pixel 481 // aspect ratio and rotation. The final size may be scaled down by video 482 // adapter to better match ratio_w_ x ratio_h_. 483 // Note that abs() of frame height is passed in, because source may be 484 // inverted, but output will be positive. 485 int desired_width = captured_frame->width; 486 int desired_height = captured_frame->height; 487 488 // TODO(fbarchard): Improve logic to pad or crop. 489 // MJPG can crop vertically, but not horizontally. This logic disables crop. 490 // Alternatively we could pad the image with black, or implement a 2 step 491 // crop. 492 bool can_crop = true; 493 if (captured_frame->fourcc == FOURCC_MJPG) { 494 float cam_aspect = static_cast<float>(captured_frame->width) / 495 static_cast<float>(captured_frame->height); 496 float view_aspect = static_cast<float>(ratio_w_) / 497 static_cast<float>(ratio_h_); 498 can_crop = cam_aspect <= view_aspect; 499 } 500 if (can_crop && !IsScreencast()) { 501 // TODO(ronghuawu): The capturer should always produce the native 502 // resolution and the cropping should be done in downstream code. 503 ComputeCrop(ratio_w_, ratio_h_, captured_frame->width, 504 abs(captured_frame->height), captured_frame->pixel_width, 505 captured_frame->pixel_height, captured_frame->rotation, 506 &desired_width, &desired_height); 507 } 508 509 if (!frame_factory_) { 510 LOG(LS_ERROR) << "No video frame factory."; 511 return; 512 } 513 514 rtc::scoped_ptr<VideoFrame> i420_frame( 515 frame_factory_->CreateAliasedFrame( 516 captured_frame, desired_width, desired_height)); 517 if (!i420_frame) { 518 // TODO(fbarchard): LOG more information about captured frame attributes. 519 LOG(LS_ERROR) << "Couldn't convert to I420! " 520 << "From " << ToString(captured_frame) << " To " 521 << desired_width << " x " << desired_height; 522 return; 523 } 524 525 VideoFrame* adapted_frame = i420_frame.get(); 526 if (enable_video_adapter_ && !IsScreencast()) { 527 VideoFrame* out_frame = NULL; 528 video_adapter_.AdaptFrame(adapted_frame, &out_frame); 529 if (!out_frame) { 530 // VideoAdapter dropped the frame. 531 ++adapt_frame_drops_; 532 return; 533 } 534 adapted_frame = out_frame; 535 } 536 537 if (!muted_ && !ApplyProcessors(adapted_frame)) { 538 // Processor dropped the frame. 539 ++effect_frame_drops_; 540 return; 541 } 542 if (muted_) { 543 // TODO(pthatcher): Use frame_factory_->CreateBlackFrame() instead. 544 adapted_frame->SetToBlack(); 545 } 546 SignalVideoFrame(this, adapted_frame); 547 548 UpdateStats(captured_frame); 549 } 550 551 void VideoCapturer::SetCaptureState(CaptureState state) { 552 if (state == capture_state_) { 553 // Don't trigger a state changed callback if the state hasn't changed. 554 return; 555 } 556 StateChangeParams* state_params = new StateChangeParams(state); 557 capture_state_ = state; 558 thread_->Post(this, MSG_STATE_CHANGE, state_params); 559 } 560 561 void VideoCapturer::OnMessage(rtc::Message* message) { 562 switch (message->message_id) { 563 case MSG_STATE_CHANGE: { 564 rtc::scoped_ptr<StateChangeParams> p( 565 static_cast<StateChangeParams*>(message->pdata)); 566 SignalStateChange(this, p->data()); 567 break; 568 } 569 case MSG_DO_PAUSE: { 570 Pause(true); 571 break; 572 } 573 case MSG_DO_UNPAUSE: { 574 Pause(false); 575 break; 576 } 577 default: { 578 ASSERT(false); 579 } 580 } 581 } 582 583 // Get the distance between the supported and desired formats. 584 // Prioritization is done according to this algorithm: 585 // 1) Width closeness. If not same, we prefer wider. 586 // 2) Height closeness. If not same, we prefer higher. 587 // 3) Framerate closeness. If not same, we prefer faster. 588 // 4) Compression. If desired format has a specific fourcc, we need exact match; 589 // otherwise, we use preference. 590 int64 VideoCapturer::GetFormatDistance(const VideoFormat& desired, 591 const VideoFormat& supported) { 592 int64 distance = kMaxDistance; 593 594 // Check fourcc. 595 uint32 supported_fourcc = CanonicalFourCC(supported.fourcc); 596 int64 delta_fourcc = kMaxDistance; 597 if (FOURCC_ANY == desired.fourcc) { 598 // Any fourcc is OK for the desired. Use preference to find best fourcc. 599 std::vector<uint32> preferred_fourccs; 600 if (!GetPreferredFourccs(&preferred_fourccs)) { 601 return distance; 602 } 603 604 for (size_t i = 0; i < preferred_fourccs.size(); ++i) { 605 if (supported_fourcc == CanonicalFourCC(preferred_fourccs[i])) { 606 delta_fourcc = i; 607 #ifdef LINUX 608 // For HD avoid YU12 which is a software conversion and has 2 bugs 609 // b/7326348 b/6960899. Reenable when fixed. 610 if (supported.height >= 720 && (supported_fourcc == FOURCC_YU12 || 611 supported_fourcc == FOURCC_YV12)) { 612 delta_fourcc += kYU12Penalty; 613 } 614 #endif 615 break; 616 } 617 } 618 } else if (supported_fourcc == CanonicalFourCC(desired.fourcc)) { 619 delta_fourcc = 0; // Need exact match. 620 } 621 622 if (kMaxDistance == delta_fourcc) { 623 // Failed to match fourcc. 624 return distance; 625 } 626 627 // Check resolution and fps. 628 int desired_width = desired.width; 629 int desired_height = desired.height; 630 int64 delta_w = supported.width - desired_width; 631 float supported_fps = VideoFormat::IntervalToFpsFloat(supported.interval); 632 float delta_fps = 633 supported_fps - VideoFormat::IntervalToFpsFloat(desired.interval); 634 // Check height of supported height compared to height we would like it to be. 635 int64 aspect_h = 636 desired_width ? supported.width * desired_height / desired_width 637 : desired_height; 638 int64 delta_h = supported.height - aspect_h; 639 640 distance = 0; 641 // Set high penalty if the supported format is lower than the desired format. 642 // 3x means we would prefer down to down to 3/4, than up to double. 643 // But we'd prefer up to double than down to 1/2. This is conservative, 644 // strongly avoiding going down in resolution, similar to 645 // the old method, but not completely ruling it out in extreme situations. 646 // It also ignores framerate, which is often very low at high resolutions. 647 // TODO(fbarchard): Improve logic to use weighted factors. 648 static const int kDownPenalty = -3; 649 if (delta_w < 0) { 650 delta_w = delta_w * kDownPenalty; 651 } 652 if (delta_h < 0) { 653 delta_h = delta_h * kDownPenalty; 654 } 655 // Require camera fps to be at least 80% of what is requested if resolution 656 // matches. 657 // Require camera fps to be at least 96% of what is requested, or higher, 658 // if resolution differs. 96% allows for slight variations in fps. e.g. 29.97 659 if (delta_fps < 0) { 660 float min_desirable_fps = delta_w ? 661 VideoFormat::IntervalToFpsFloat(desired.interval) * 28.f / 30.f : 662 VideoFormat::IntervalToFpsFloat(desired.interval) * 23.f / 30.f; 663 delta_fps = -delta_fps; 664 if (supported_fps < min_desirable_fps) { 665 distance |= static_cast<int64>(1) << 62; 666 } else { 667 distance |= static_cast<int64>(1) << 15; 668 } 669 } 670 int64 idelta_fps = static_cast<int>(delta_fps); 671 672 // 12 bits for width and height and 8 bits for fps and fourcc. 673 distance |= 674 (delta_w << 28) | (delta_h << 16) | (idelta_fps << 8) | delta_fourcc; 675 676 return distance; 677 } 678 679 bool VideoCapturer::ApplyProcessors(VideoFrame* video_frame) { 680 bool drop_frame = false; 681 rtc::CritScope cs(&crit_); 682 for (VideoProcessors::iterator iter = video_processors_.begin(); 683 iter != video_processors_.end(); ++iter) { 684 (*iter)->OnFrame(kDummyVideoSsrc, video_frame, &drop_frame); 685 if (drop_frame) { 686 return false; 687 } 688 } 689 return true; 690 } 691 692 void VideoCapturer::UpdateFilteredSupportedFormats() { 693 filtered_supported_formats_.clear(); 694 filtered_supported_formats_ = supported_formats_; 695 if (!max_format_) { 696 return; 697 } 698 std::vector<VideoFormat>::iterator iter = filtered_supported_formats_.begin(); 699 while (iter != filtered_supported_formats_.end()) { 700 if (ShouldFilterFormat(*iter)) { 701 iter = filtered_supported_formats_.erase(iter); 702 } else { 703 ++iter; 704 } 705 } 706 if (filtered_supported_formats_.empty()) { 707 // The device only captures at resolutions higher than |max_format_| this 708 // indicates that |max_format_| should be ignored as it is better to capture 709 // at too high a resolution than to not capture at all. 710 filtered_supported_formats_ = supported_formats_; 711 } 712 } 713 714 bool VideoCapturer::ShouldFilterFormat(const VideoFormat& format) const { 715 if (!enable_camera_list_) { 716 return false; 717 } 718 return format.width > max_format_->width || 719 format.height > max_format_->height; 720 } 721 722 void VideoCapturer::UpdateStats(const CapturedFrame* captured_frame) { 723 // Update stats protected from fetches from different thread. 724 rtc::CritScope cs(&frame_stats_crit_); 725 726 last_captured_frame_format_.width = captured_frame->width; 727 last_captured_frame_format_.height = captured_frame->height; 728 // TODO(ronghuawu): Useful to report interval as well? 729 last_captured_frame_format_.interval = 0; 730 last_captured_frame_format_.fourcc = captured_frame->fourcc; 731 732 double time_now = frame_length_time_reporter_.TimerNow(); 733 if (previous_frame_time_ != 0.0) { 734 adapt_frame_drops_data_.AddSample(adapt_frame_drops_); 735 effect_frame_drops_data_.AddSample(effect_frame_drops_); 736 frame_time_data_.AddSample(time_now - previous_frame_time_); 737 } 738 previous_frame_time_ = time_now; 739 effect_frame_drops_ = 0; 740 adapt_frame_drops_ = 0; 741 } 742 743 template<class T> 744 void VideoCapturer::GetVariableSnapshot( 745 const rtc::RollingAccumulator<T>& data, 746 VariableInfo<T>* stats) { 747 stats->max_val = data.ComputeMax(); 748 stats->mean = data.ComputeMean(); 749 stats->min_val = data.ComputeMin(); 750 stats->variance = data.ComputeVariance(); 751 } 752 753 } // namespace cricket 754
