1 // libjingle 2 // Copyright 2010 Google Inc. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are met: 6 // 7 // 1. Redistributions of source code must retain the above copyright notice, 8 // this list of conditions and the following disclaimer. 9 // 2. Redistributions in binary form must reproduce the above copyright notice, 10 // this list of conditions and the following disclaimer in the documentation 11 // and/or other materials provided with the distribution. 12 // 3. The name of the author may not be used to endorse or promote products 13 // derived from this software without specific prior written permission. 14 // 15 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 16 // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 17 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 18 // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 19 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 24 // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 // 26 // Implementation file of class VideoCapturer. 27 28 #include "talk/media/base/videocapturer.h" 29 30 #include <algorithm> 31 32 #if !defined(DISABLE_YUV) 33 #include "libyuv/scale_argb.h" 34 #endif 35 #include "talk/base/common.h" 36 #include "talk/base/logging.h" 37 #include "talk/base/systeminfo.h" 38 #include "talk/media/base/videoprocessor.h" 39 40 #if defined(HAVE_WEBRTC_VIDEO) 41 #include "talk/media/webrtc/webrtcvideoframe.h" 42 #endif // HAVE_WEBRTC_VIDEO 43 44 45 namespace cricket { 46 47 namespace { 48 49 // TODO(thorcarpenter): This is a BIG hack to flush the system with black 50 // frames. Frontends should coordinate to update the video state of a muted 51 // user. When all frontends to this consider removing the black frame business. 52 const int kNumBlackFramesOnMute = 30; 53 54 // MessageHandler constants. 55 enum { 56 MSG_DO_PAUSE = 0, 57 MSG_DO_UNPAUSE, 58 MSG_STATE_CHANGE 59 }; 60 61 static const int64 kMaxDistance = ~(static_cast<int64>(1) << 63); 62 #ifdef LINUX 63 static const int kYU12Penalty = 16; // Needs to be higher than MJPG index. 64 #endif 65 static const int kDefaultScreencastFps = 5; 66 typedef talk_base::TypedMessageData<CaptureState> StateChangeParams; 67 68 // Limit stats data collections to ~20 seconds of 30fps data before dropping 69 // old data in case stats aren't reset for long periods of time. 70 static const size_t kMaxAccumulatorSize = 600; 71 72 } // namespace 73 74 ///////////////////////////////////////////////////////////////////// 75 // Implementation of struct CapturedFrame 76 ///////////////////////////////////////////////////////////////////// 77 CapturedFrame::CapturedFrame() 78 : width(0), 79 height(0), 80 fourcc(0), 81 pixel_width(0), 82 pixel_height(0), 83 elapsed_time(0), 84 time_stamp(0), 85 data_size(0), 86 rotation(0), 87 data(NULL) {} 88 89 // TODO(fbarchard): Remove this function once lmimediaengine stops using it. 90 bool CapturedFrame::GetDataSize(uint32* size) const { 91 if (!size || data_size == CapturedFrame::kUnknownDataSize) { 92 return false; 93 } 94 *size = data_size; 95 return true; 96 } 97 98 ///////////////////////////////////////////////////////////////////// 99 // Implementation of class VideoCapturer 100 ///////////////////////////////////////////////////////////////////// 101 VideoCapturer::VideoCapturer() 102 : thread_(talk_base::Thread::Current()), 103 adapt_frame_drops_data_(kMaxAccumulatorSize), 104 effect_frame_drops_data_(kMaxAccumulatorSize), 105 frame_time_data_(kMaxAccumulatorSize) { 106 Construct(); 107 } 108 109 VideoCapturer::VideoCapturer(talk_base::Thread* thread) 110 : thread_(thread), 111 adapt_frame_drops_data_(kMaxAccumulatorSize), 112 effect_frame_drops_data_(kMaxAccumulatorSize), 113 frame_time_data_(kMaxAccumulatorSize) { 114 Construct(); 115 } 116 117 void VideoCapturer::Construct() { 118 ClearAspectRatio(); 119 enable_camera_list_ = false; 120 square_pixel_aspect_ratio_ = false; 121 capture_state_ = CS_STOPPED; 122 SignalFrameCaptured.connect(this, &VideoCapturer::OnFrameCaptured); 123 scaled_width_ = 0; 124 scaled_height_ = 0; 125 screencast_max_pixels_ = 0; 126 muted_ = false; 127 black_frame_count_down_ = kNumBlackFramesOnMute; 128 enable_video_adapter_ = true; 129 adapt_frame_drops_ = 0; 130 effect_frame_drops_ = 0; 131 previous_frame_time_ = 0.0; 132 } 133 134 const std::vector<VideoFormat>* VideoCapturer::GetSupportedFormats() const { 135 return &filtered_supported_formats_; 136 } 137 138 bool VideoCapturer::StartCapturing(const VideoFormat& capture_format) { 139 previous_frame_time_ = frame_length_time_reporter_.TimerNow(); 140 CaptureState result = Start(capture_format); 141 const bool success = (result == CS_RUNNING) || (result == CS_STARTING); 142 if (!success) { 143 return false; 144 } 145 if (result == CS_RUNNING) { 146 SetCaptureState(result); 147 } 148 return true; 149 } 150 151 void VideoCapturer::UpdateAspectRatio(int ratio_w, int ratio_h) { 152 if (ratio_w == 0 || ratio_h == 0) { 153 LOG(LS_WARNING) << "UpdateAspectRatio ignored invalid ratio: " 154 << ratio_w << "x" << ratio_h; 155 return; 156 } 157 ratio_w_ = ratio_w; 158 ratio_h_ = ratio_h; 159 } 160 161 void VideoCapturer::ClearAspectRatio() { 162 ratio_w_ = 0; 163 ratio_h_ = 0; 164 } 165 166 // Override this to have more control of how your device is started/stopped. 167 bool VideoCapturer::Pause(bool pause) { 168 if (pause) { 169 if (capture_state() == CS_PAUSED) { 170 return true; 171 } 172 bool is_running = capture_state() == CS_STARTING || 173 capture_state() == CS_RUNNING; 174 if (!is_running) { 175 LOG(LS_ERROR) << "Cannot pause a stopped camera."; 176 return false; 177 } 178 LOG(LS_INFO) << "Pausing a camera."; 179 talk_base::scoped_ptr<VideoFormat> capture_format_when_paused( 180 capture_format_ ? new VideoFormat(*capture_format_) : NULL); 181 Stop(); 182 SetCaptureState(CS_PAUSED); 183 // If you override this function be sure to restore the capture format 184 // after calling Stop(). 185 SetCaptureFormat(capture_format_when_paused.get()); 186 } else { // Unpause. 187 if (capture_state() != CS_PAUSED) { 188 LOG(LS_WARNING) << "Cannot unpause a camera that hasn't been paused."; 189 return false; 190 } 191 if (!capture_format_) { 192 LOG(LS_ERROR) << "Missing capture_format_, cannot unpause a camera."; 193 return false; 194 } 195 if (muted_) { 196 LOG(LS_WARNING) << "Camera cannot be unpaused while muted."; 197 return false; 198 } 199 LOG(LS_INFO) << "Unpausing a camera."; 200 if (!Start(*capture_format_)) { 201 LOG(LS_ERROR) << "Camera failed to start when unpausing."; 202 return false; 203 } 204 } 205 return true; 206 } 207 208 bool VideoCapturer::Restart(const VideoFormat& capture_format) { 209 if (!IsRunning()) { 210 return StartCapturing(capture_format); 211 } 212 213 if (GetCaptureFormat() != NULL && *GetCaptureFormat() == capture_format) { 214 // The reqested format is the same; nothing to do. 215 return true; 216 } 217 218 Stop(); 219 return StartCapturing(capture_format); 220 } 221 222 bool VideoCapturer::MuteToBlackThenPause(bool muted) { 223 if (muted == IsMuted()) { 224 return true; 225 } 226 227 LOG(LS_INFO) << (muted ? "Muting" : "Unmuting") << " this video capturer."; 228 muted_ = muted; // Do this before calling Pause(). 229 if (muted) { 230 // Reset black frame count down. 231 black_frame_count_down_ = kNumBlackFramesOnMute; 232 // Following frames will be overritten with black, then the camera will be 233 // paused. 234 return true; 235 } 236 // Start the camera. 237 thread_->Clear(this, MSG_DO_PAUSE); 238 return Pause(false); 239 } 240 241 void VideoCapturer::SetSupportedFormats( 242 const std::vector<VideoFormat>& formats) { 243 supported_formats_ = formats; 244 UpdateFilteredSupportedFormats(); 245 } 246 247 bool VideoCapturer::GetBestCaptureFormat(const VideoFormat& format, 248 VideoFormat* best_format) { 249 // TODO(fbarchard): Directly support max_format. 250 UpdateFilteredSupportedFormats(); 251 const std::vector<VideoFormat>* supported_formats = GetSupportedFormats(); 252 253 if (supported_formats->empty()) { 254 return false; 255 } 256 LOG(LS_INFO) << " Capture Requested " << format.ToString(); 257 int64 best_distance = kMaxDistance; 258 std::vector<VideoFormat>::const_iterator best = supported_formats->end(); 259 std::vector<VideoFormat>::const_iterator i; 260 for (i = supported_formats->begin(); i != supported_formats->end(); ++i) { 261 int64 distance = GetFormatDistance(format, *i); 262 // TODO(fbarchard): Reduce to LS_VERBOSE if/when camera capture is 263 // relatively bug free. 264 LOG(LS_INFO) << " Supported " << i->ToString() << " distance " << distance; 265 if (distance < best_distance) { 266 best_distance = distance; 267 best = i; 268 } 269 } 270 if (supported_formats->end() == best) { 271 LOG(LS_ERROR) << " No acceptable camera format found"; 272 return false; 273 } 274 275 if (best_format) { 276 best_format->width = best->width; 277 best_format->height = best->height; 278 best_format->fourcc = best->fourcc; 279 best_format->interval = best->interval; 280 LOG(LS_INFO) << " Best " << best_format->ToString() << " Interval " 281 << best_format->interval << " distance " << best_distance; 282 } 283 return true; 284 } 285 286 void VideoCapturer::AddVideoProcessor(VideoProcessor* video_processor) { 287 talk_base::CritScope cs(&crit_); 288 ASSERT(std::find(video_processors_.begin(), video_processors_.end(), 289 video_processor) == video_processors_.end()); 290 video_processors_.push_back(video_processor); 291 } 292 293 bool VideoCapturer::RemoveVideoProcessor(VideoProcessor* video_processor) { 294 talk_base::CritScope cs(&crit_); 295 VideoProcessors::iterator found = std::find( 296 video_processors_.begin(), video_processors_.end(), video_processor); 297 if (found == video_processors_.end()) { 298 return false; 299 } 300 video_processors_.erase(found); 301 return true; 302 } 303 304 void VideoCapturer::ConstrainSupportedFormats(const VideoFormat& max_format) { 305 max_format_.reset(new VideoFormat(max_format)); 306 LOG(LS_VERBOSE) << " ConstrainSupportedFormats " << max_format.ToString(); 307 UpdateFilteredSupportedFormats(); 308 } 309 310 std::string VideoCapturer::ToString(const CapturedFrame* captured_frame) const { 311 std::string fourcc_name = GetFourccName(captured_frame->fourcc) + " "; 312 for (std::string::const_iterator i = fourcc_name.begin(); 313 i < fourcc_name.end(); ++i) { 314 // Test character is printable; Avoid isprint() which asserts on negatives. 315 if (*i < 32 || *i >= 127) { 316 fourcc_name = ""; 317 break; 318 } 319 } 320 321 std::ostringstream ss; 322 ss << fourcc_name << captured_frame->width << "x" << captured_frame->height 323 << "x" << VideoFormat::IntervalToFpsFloat(captured_frame->elapsed_time); 324 return ss.str(); 325 } 326 327 void VideoCapturer::GetStats(VariableInfo<int>* adapt_drops_stats, 328 VariableInfo<int>* effect_drops_stats, 329 VariableInfo<double>* frame_time_stats, 330 VideoFormat* last_captured_frame_format) { 331 talk_base::CritScope cs(&frame_stats_crit_); 332 GetVariableSnapshot(adapt_frame_drops_data_, adapt_drops_stats); 333 GetVariableSnapshot(effect_frame_drops_data_, effect_drops_stats); 334 GetVariableSnapshot(frame_time_data_, frame_time_stats); 335 *last_captured_frame_format = last_captured_frame_format_; 336 337 adapt_frame_drops_data_.Reset(); 338 effect_frame_drops_data_.Reset(); 339 frame_time_data_.Reset(); 340 } 341 342 void VideoCapturer::OnFrameCaptured(VideoCapturer*, 343 const CapturedFrame* captured_frame) { 344 if (muted_) { 345 if (black_frame_count_down_ == 0) { 346 thread_->Post(this, MSG_DO_PAUSE, NULL); 347 } else { 348 --black_frame_count_down_; 349 } 350 } 351 352 if (SignalVideoFrame.is_empty()) { 353 return; 354 } 355 #if defined(HAVE_WEBRTC_VIDEO) 356 #define VIDEO_FRAME_NAME WebRtcVideoFrame 357 #endif 358 #if defined(VIDEO_FRAME_NAME) 359 #if !defined(DISABLE_YUV) 360 if (IsScreencast()) { 361 int scaled_width, scaled_height; 362 if (screencast_max_pixels_ > 0) { 363 ComputeScaleMaxPixels(captured_frame->width, captured_frame->height, 364 screencast_max_pixels_, &scaled_width, &scaled_height); 365 } else { 366 int desired_screencast_fps = capture_format_.get() ? 367 VideoFormat::IntervalToFps(capture_format_->interval) : 368 kDefaultScreencastFps; 369 ComputeScale(captured_frame->width, captured_frame->height, 370 desired_screencast_fps, &scaled_width, &scaled_height); 371 } 372 373 if (FOURCC_ARGB == captured_frame->fourcc && 374 (scaled_width != captured_frame->width || 375 scaled_height != captured_frame->height)) { 376 if (scaled_width != scaled_width_ || scaled_height != scaled_height_) { 377 LOG(LS_INFO) << "Scaling Screencast from " 378 << captured_frame->width << "x" 379 << captured_frame->height << " to " 380 << scaled_width << "x" << scaled_height; 381 scaled_width_ = scaled_width; 382 scaled_height_ = scaled_height; 383 } 384 CapturedFrame* modified_frame = 385 const_cast<CapturedFrame*>(captured_frame); 386 // Compute new width such that width * height is less than maximum but 387 // maintains original captured frame aspect ratio. 388 // Round down width to multiple of 4 so odd width won't round up beyond 389 // maximum, and so chroma channel is even width to simplify spatial 390 // resampling. 391 libyuv::ARGBScale(reinterpret_cast<const uint8*>(captured_frame->data), 392 captured_frame->width * 4, captured_frame->width, 393 captured_frame->height, 394 reinterpret_cast<uint8*>(modified_frame->data), 395 scaled_width * 4, scaled_width, scaled_height, 396 libyuv::kFilterBilinear); 397 modified_frame->width = scaled_width; 398 modified_frame->height = scaled_height; 399 modified_frame->data_size = scaled_width * 4 * scaled_height; 400 } 401 } 402 403 const int kYuy2Bpp = 2; 404 const int kArgbBpp = 4; 405 // TODO(fbarchard): Make a helper function to adjust pixels to square. 406 // TODO(fbarchard): Hook up experiment to scaling. 407 // TODO(fbarchard): Avoid scale and convert if muted. 408 // Temporary buffer is scoped here so it will persist until i420_frame.Init() 409 // makes a copy of the frame, converting to I420. 410 talk_base::scoped_ptr<uint8[]> temp_buffer; 411 // YUY2 can be scaled vertically using an ARGB scaler. Aspect ratio is only 412 // a problem on OSX. OSX always converts webcams to YUY2 or UYVY. 413 bool can_scale = 414 FOURCC_YUY2 == CanonicalFourCC(captured_frame->fourcc) || 415 FOURCC_UYVY == CanonicalFourCC(captured_frame->fourcc); 416 417 // If pixels are not square, optionally use vertical scaling to make them 418 // square. Square pixels simplify the rest of the pipeline, including 419 // effects and rendering. 420 if (can_scale && square_pixel_aspect_ratio_ && 421 captured_frame->pixel_width != captured_frame->pixel_height) { 422 int scaled_width, scaled_height; 423 // modified_frame points to the captured_frame but with const casted away 424 // so it can be modified. 425 CapturedFrame* modified_frame = const_cast<CapturedFrame*>(captured_frame); 426 // Compute the frame size that makes pixels square pixel aspect ratio. 427 ComputeScaleToSquarePixels(captured_frame->width, captured_frame->height, 428 captured_frame->pixel_width, 429 captured_frame->pixel_height, 430 &scaled_width, &scaled_height); 431 432 if (scaled_width != scaled_width_ || scaled_height != scaled_height_) { 433 LOG(LS_INFO) << "Scaling WebCam from " 434 << captured_frame->width << "x" 435 << captured_frame->height << " to " 436 << scaled_width << "x" << scaled_height 437 << " for PAR " 438 << captured_frame->pixel_width << "x" 439 << captured_frame->pixel_height; 440 scaled_width_ = scaled_width; 441 scaled_height_ = scaled_height; 442 } 443 const int modified_frame_size = scaled_width * scaled_height * kYuy2Bpp; 444 uint8* temp_buffer_data; 445 // Pixels are wide and short; Increasing height. Requires temporary buffer. 446 if (scaled_height > captured_frame->height) { 447 temp_buffer.reset(new uint8[modified_frame_size]); 448 temp_buffer_data = temp_buffer.get(); 449 } else { 450 // Pixels are narrow and tall; Decreasing height. Scale will be done 451 // in place. 452 temp_buffer_data = reinterpret_cast<uint8*>(captured_frame->data); 453 } 454 455 // Use ARGBScaler to vertically scale the YUY2 image, adjusting for 16 bpp. 456 libyuv::ARGBScale(reinterpret_cast<const uint8*>(captured_frame->data), 457 captured_frame->width * kYuy2Bpp, // Stride for YUY2. 458 captured_frame->width * kYuy2Bpp / kArgbBpp, // Width. 459 abs(captured_frame->height), // Height. 460 temp_buffer_data, 461 scaled_width * kYuy2Bpp, // Stride for YUY2. 462 scaled_width * kYuy2Bpp / kArgbBpp, // Width. 463 abs(scaled_height), // New height. 464 libyuv::kFilterBilinear); 465 modified_frame->width = scaled_width; 466 modified_frame->height = scaled_height; 467 modified_frame->pixel_width = 1; 468 modified_frame->pixel_height = 1; 469 modified_frame->data_size = modified_frame_size; 470 modified_frame->data = temp_buffer_data; 471 } 472 #endif // !DISABLE_YUV 473 474 // Size to crop captured frame to. This adjusts the captured frames 475 // aspect ratio to match the final view aspect ratio, considering pixel 476 // aspect ratio and rotation. The final size may be scaled down by video 477 // adapter to better match ratio_w_ x ratio_h_. 478 // Note that abs() of frame height is passed in, because source may be 479 // inverted, but output will be positive. 480 int desired_width = captured_frame->width; 481 int desired_height = captured_frame->height; 482 483 // TODO(fbarchard): Improve logic to pad or crop. 484 // MJPG can crop vertically, but not horizontally. This logic disables crop. 485 // Alternatively we could pad the image with black, or implement a 2 step 486 // crop. 487 bool can_crop = true; 488 if (captured_frame->fourcc == FOURCC_MJPG) { 489 float cam_aspect = static_cast<float>(captured_frame->width) / 490 static_cast<float>(captured_frame->height); 491 float view_aspect = static_cast<float>(ratio_w_) / 492 static_cast<float>(ratio_h_); 493 can_crop = cam_aspect <= view_aspect; 494 } 495 if (can_crop && !IsScreencast()) { 496 // TODO(ronghuawu): The capturer should always produce the native 497 // resolution and the cropping should be done in downstream code. 498 ComputeCrop(ratio_w_, ratio_h_, captured_frame->width, 499 abs(captured_frame->height), captured_frame->pixel_width, 500 captured_frame->pixel_height, captured_frame->rotation, 501 &desired_width, &desired_height); 502 } 503 504 VIDEO_FRAME_NAME i420_frame; 505 if (!i420_frame.Alias(captured_frame, desired_width, desired_height)) { 506 // TODO(fbarchard): LOG more information about captured frame attributes. 507 LOG(LS_ERROR) << "Couldn't convert to I420! " 508 << "From " << ToString(captured_frame) << " To " 509 << desired_width << " x " << desired_height; 510 return; 511 } 512 513 VideoFrame* adapted_frame = &i420_frame; 514 if (enable_video_adapter_ && !IsScreencast()) { 515 VideoFrame* out_frame = NULL; 516 video_adapter_.AdaptFrame(adapted_frame, &out_frame); 517 if (!out_frame) { 518 // VideoAdapter dropped the frame. 519 ++adapt_frame_drops_; 520 return; 521 } 522 adapted_frame = out_frame; 523 } 524 525 if (!muted_ && !ApplyProcessors(adapted_frame)) { 526 // Processor dropped the frame. 527 ++effect_frame_drops_; 528 return; 529 } 530 if (muted_) { 531 adapted_frame->SetToBlack(); 532 } 533 SignalVideoFrame(this, adapted_frame); 534 535 UpdateStats(captured_frame); 536 537 #endif // VIDEO_FRAME_NAME 538 } 539 540 void VideoCapturer::SetCaptureState(CaptureState state) { 541 if (state == capture_state_) { 542 // Don't trigger a state changed callback if the state hasn't changed. 543 return; 544 } 545 StateChangeParams* state_params = new StateChangeParams(state); 546 capture_state_ = state; 547 thread_->Post(this, MSG_STATE_CHANGE, state_params); 548 } 549 550 void VideoCapturer::OnMessage(talk_base::Message* message) { 551 switch (message->message_id) { 552 case MSG_STATE_CHANGE: { 553 talk_base::scoped_ptr<StateChangeParams> p( 554 static_cast<StateChangeParams*>(message->pdata)); 555 SignalStateChange(this, p->data()); 556 break; 557 } 558 case MSG_DO_PAUSE: { 559 Pause(true); 560 break; 561 } 562 case MSG_DO_UNPAUSE: { 563 Pause(false); 564 break; 565 } 566 default: { 567 ASSERT(false); 568 } 569 } 570 } 571 572 // Get the distance between the supported and desired formats. 573 // Prioritization is done according to this algorithm: 574 // 1) Width closeness. If not same, we prefer wider. 575 // 2) Height closeness. If not same, we prefer higher. 576 // 3) Framerate closeness. If not same, we prefer faster. 577 // 4) Compression. If desired format has a specific fourcc, we need exact match; 578 // otherwise, we use preference. 579 int64 VideoCapturer::GetFormatDistance(const VideoFormat& desired, 580 const VideoFormat& supported) { 581 int64 distance = kMaxDistance; 582 583 // Check fourcc. 584 uint32 supported_fourcc = CanonicalFourCC(supported.fourcc); 585 int64 delta_fourcc = kMaxDistance; 586 if (FOURCC_ANY == desired.fourcc) { 587 // Any fourcc is OK for the desired. Use preference to find best fourcc. 588 std::vector<uint32> preferred_fourccs; 589 if (!GetPreferredFourccs(&preferred_fourccs)) { 590 return distance; 591 } 592 593 for (size_t i = 0; i < preferred_fourccs.size(); ++i) { 594 if (supported_fourcc == CanonicalFourCC(preferred_fourccs[i])) { 595 delta_fourcc = i; 596 #ifdef LINUX 597 // For HD avoid YU12 which is a software conversion and has 2 bugs 598 // b/7326348 b/6960899. Reenable when fixed. 599 if (supported.height >= 720 && (supported_fourcc == FOURCC_YU12 || 600 supported_fourcc == FOURCC_YV12)) { 601 delta_fourcc += kYU12Penalty; 602 } 603 #endif 604 break; 605 } 606 } 607 } else if (supported_fourcc == CanonicalFourCC(desired.fourcc)) { 608 delta_fourcc = 0; // Need exact match. 609 } 610 611 if (kMaxDistance == delta_fourcc) { 612 // Failed to match fourcc. 613 return distance; 614 } 615 616 // Check resolution and fps. 617 int desired_width = desired.width; 618 int desired_height = desired.height; 619 int64 delta_w = supported.width - desired_width; 620 float supported_fps = VideoFormat::IntervalToFpsFloat(supported.interval); 621 float delta_fps = 622 supported_fps - VideoFormat::IntervalToFpsFloat(desired.interval); 623 // Check height of supported height compared to height we would like it to be. 624 int64 aspect_h = 625 desired_width ? supported.width * desired_height / desired_width 626 : desired_height; 627 int64 delta_h = supported.height - aspect_h; 628 629 distance = 0; 630 // Set high penalty if the supported format is lower than the desired format. 631 // 3x means we would prefer down to down to 3/4, than up to double. 632 // But we'd prefer up to double than down to 1/2. This is conservative, 633 // strongly avoiding going down in resolution, similar to 634 // the old method, but not completely ruling it out in extreme situations. 635 // It also ignores framerate, which is often very low at high resolutions. 636 // TODO(fbarchard): Improve logic to use weighted factors. 637 static const int kDownPenalty = -3; 638 if (delta_w < 0) { 639 delta_w = delta_w * kDownPenalty; 640 } 641 if (delta_h < 0) { 642 delta_h = delta_h * kDownPenalty; 643 } 644 // Require camera fps to be at least 80% of what is requested if resolution 645 // matches. 646 // Require camera fps to be at least 96% of what is requested, or higher, 647 // if resolution differs. 96% allows for slight variations in fps. e.g. 29.97 648 if (delta_fps < 0) { 649 float min_desirable_fps = delta_w ? 650 VideoFormat::IntervalToFpsFloat(desired.interval) * 28.f / 30.f : 651 VideoFormat::IntervalToFpsFloat(desired.interval) * 23.f / 30.f; 652 delta_fps = -delta_fps; 653 if (supported_fps < min_desirable_fps) { 654 distance |= static_cast<int64>(1) << 62; 655 } else { 656 distance |= static_cast<int64>(1) << 15; 657 } 658 } 659 int64 idelta_fps = static_cast<int>(delta_fps); 660 661 // 12 bits for width and height and 8 bits for fps and fourcc. 662 distance |= 663 (delta_w << 28) | (delta_h << 16) | (idelta_fps << 8) | delta_fourcc; 664 665 return distance; 666 } 667 668 bool VideoCapturer::ApplyProcessors(VideoFrame* video_frame) { 669 bool drop_frame = false; 670 talk_base::CritScope cs(&crit_); 671 for (VideoProcessors::iterator iter = video_processors_.begin(); 672 iter != video_processors_.end(); ++iter) { 673 (*iter)->OnFrame(kDummyVideoSsrc, video_frame, &drop_frame); 674 if (drop_frame) { 675 return false; 676 } 677 } 678 return true; 679 } 680 681 void VideoCapturer::UpdateFilteredSupportedFormats() { 682 filtered_supported_formats_.clear(); 683 filtered_supported_formats_ = supported_formats_; 684 if (!max_format_) { 685 return; 686 } 687 std::vector<VideoFormat>::iterator iter = filtered_supported_formats_.begin(); 688 while (iter != filtered_supported_formats_.end()) { 689 if (ShouldFilterFormat(*iter)) { 690 iter = filtered_supported_formats_.erase(iter); 691 } else { 692 ++iter; 693 } 694 } 695 if (filtered_supported_formats_.empty()) { 696 // The device only captures at resolutions higher than |max_format_| this 697 // indicates that |max_format_| should be ignored as it is better to capture 698 // at too high a resolution than to not capture at all. 699 filtered_supported_formats_ = supported_formats_; 700 } 701 } 702 703 bool VideoCapturer::ShouldFilterFormat(const VideoFormat& format) const { 704 if (!enable_camera_list_) { 705 return false; 706 } 707 return format.width > max_format_->width || 708 format.height > max_format_->height; 709 } 710 711 void VideoCapturer::UpdateStats(const CapturedFrame* captured_frame) { 712 // Update stats protected from fetches from different thread. 713 talk_base::CritScope cs(&frame_stats_crit_); 714 715 last_captured_frame_format_.width = captured_frame->width; 716 last_captured_frame_format_.height = captured_frame->height; 717 // TODO(ronghuawu): Useful to report interval as well? 718 last_captured_frame_format_.interval = 0; 719 last_captured_frame_format_.fourcc = captured_frame->fourcc; 720 721 double time_now = frame_length_time_reporter_.TimerNow(); 722 if (previous_frame_time_ != 0.0) { 723 adapt_frame_drops_data_.AddSample(adapt_frame_drops_); 724 effect_frame_drops_data_.AddSample(effect_frame_drops_); 725 frame_time_data_.AddSample(time_now - previous_frame_time_); 726 } 727 previous_frame_time_ = time_now; 728 effect_frame_drops_ = 0; 729 adapt_frame_drops_ = 0; 730 } 731 732 template<class T> 733 void VideoCapturer::GetVariableSnapshot( 734 const talk_base::RollingAccumulator<T>& data, 735 VariableInfo<T>* stats) { 736 stats->max_val = data.ComputeMax(); 737 stats->mean = data.ComputeMean(); 738 stats->min_val = data.ComputeMin(); 739 stats->variance = data.ComputeVariance(); 740 } 741 742 } // namespace cricket 743
