1 // Copyright 2014 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "content/renderer/media/video_track_adapter.h" 6 7 #include <algorithm> 8 #include <limits> 9 #include <utility> 10 11 #include "base/bind.h" 12 #include "base/debug/trace_event.h" 13 #include "base/location.h" 14 #include "base/metrics/histogram.h" 15 #include "media/base/bind_to_current_loop.h" 16 #include "media/base/video_util.h" 17 18 namespace content { 19 20 namespace { 21 22 // Amount of frame intervals to wait before considering the source as muted, for 23 // the first frame and under normal conditions, respectively. First frame might 24 // take longer to arrive due to source startup. 25 const float kFirstFrameTimeoutInFrameIntervals = 100.0f; 26 const float kNormalFrameTimeoutInFrameIntervals = 25.0f; 27 28 // Min delta time between two frames allowed without being dropped if a max 29 // frame rate is specified. 30 const int kMinTimeInMsBetweenFrames = 5; 31 32 // Empty method used for keeping a reference to the original media::VideoFrame 33 // in VideoFrameResolutionAdapter::DeliverFrame if cropping is needed. 34 // The reference to |frame| is kept in the closure that calls this method. 35 void ReleaseOriginalFrame( 36 const scoped_refptr<media::VideoFrame>& frame) { 37 } 38 39 void ResetCallbackOnMainRenderThread( 40 scoped_ptr<VideoCaptureDeliverFrameCB> callback) { 41 // |callback| will be deleted when this exits. 42 } 43 44 } // anonymous namespace 45 46 // VideoFrameResolutionAdapter is created on and lives on 47 // on the IO-thread. It does the resolution adaptation and delivers frames to 48 // all registered tracks on the IO-thread. 49 // All method calls must be on the IO-thread. 50 class VideoTrackAdapter::VideoFrameResolutionAdapter 51 : public base::RefCountedThreadSafe<VideoFrameResolutionAdapter> { 52 public: 53 VideoFrameResolutionAdapter( 54 scoped_refptr<base::SingleThreadTaskRunner> render_message_loop, 55 const gfx::Size& max_size, 56 double min_aspect_ratio, 57 double max_aspect_ratio, 58 double max_frame_rate); 59 60 // Add |callback| to receive video frames on the IO-thread. 61 // |callback| will however be released on the main render thread. 62 void AddCallback(const MediaStreamVideoTrack* track, 63 const VideoCaptureDeliverFrameCB& callback); 64 65 // Removes |callback| associated with |track| from receiving video frames if 66 // |track| has been added. It is ok to call RemoveCallback even if the |track| 67 // has not been added. The |callback| is released on the main render thread. 68 void RemoveCallback(const MediaStreamVideoTrack* track); 69 70 void DeliverFrame(const scoped_refptr<media::VideoFrame>& frame, 71 const media::VideoCaptureFormat& format, 72 const base::TimeTicks& estimated_capture_time); 73 74 // Returns true if all arguments match with the output of this adapter. 75 bool ConstraintsMatch(const gfx::Size& max_size, 76 double min_aspect_ratio, 77 double max_aspect_ratio, 78 double max_frame_rate) const; 79 80 bool IsEmpty() const; 81 82 private: 83 virtual ~VideoFrameResolutionAdapter(); 84 friend class base::RefCountedThreadSafe<VideoFrameResolutionAdapter>; 85 86 virtual void DoDeliverFrame( 87 const scoped_refptr<media::VideoFrame>& frame, 88 const media::VideoCaptureFormat& format, 89 const base::TimeTicks& estimated_capture_time); 90 91 // Returns |true| if the input frame rate is higher that the requested max 92 // frame rate and |frame| should be dropped. 93 bool MaybeDropFrame(const scoped_refptr<media::VideoFrame>& frame, 94 float source_frame_rate); 95 96 // Bound to the IO-thread. 97 base::ThreadChecker io_thread_checker_; 98 99 // The task runner where we will release VideoCaptureDeliverFrameCB 100 // registered in AddCallback. 101 scoped_refptr<base::SingleThreadTaskRunner> renderer_task_runner_; 102 103 gfx::Size max_frame_size_; 104 double min_aspect_ratio_; 105 double max_aspect_ratio_; 106 107 double frame_rate_; 108 base::TimeDelta last_time_stamp_; 109 double max_frame_rate_; 110 double keep_frame_counter_; 111 112 typedef std::pair<const void*, VideoCaptureDeliverFrameCB> 113 VideoIdCallbackPair; 114 std::vector<VideoIdCallbackPair> callbacks_; 115 116 DISALLOW_COPY_AND_ASSIGN(VideoFrameResolutionAdapter); 117 }; 118 119 VideoTrackAdapter:: 120 VideoFrameResolutionAdapter::VideoFrameResolutionAdapter( 121 scoped_refptr<base::SingleThreadTaskRunner> render_message_loop, 122 const gfx::Size& max_size, 123 double min_aspect_ratio, 124 double max_aspect_ratio, 125 double max_frame_rate) 126 : renderer_task_runner_(render_message_loop), 127 max_frame_size_(max_size), 128 min_aspect_ratio_(min_aspect_ratio), 129 max_aspect_ratio_(max_aspect_ratio), 130 frame_rate_(MediaStreamVideoSource::kDefaultFrameRate), 131 max_frame_rate_(max_frame_rate), 132 keep_frame_counter_(0.0f) { 133 DCHECK(renderer_task_runner_.get()); 134 DCHECK(io_thread_checker_.CalledOnValidThread()); 135 DCHECK_GE(max_aspect_ratio_, min_aspect_ratio_); 136 CHECK_NE(0, max_aspect_ratio_); 137 DVLOG(3) << "VideoFrameResolutionAdapter(" 138 << "{ max_width =" << max_frame_size_.width() << "}, " 139 << "{ max_height =" << max_frame_size_.height() << "}, " 140 << "{ min_aspect_ratio =" << min_aspect_ratio << "}, " 141 << "{ max_aspect_ratio_ =" << max_aspect_ratio_ << "}" 142 << "{ max_frame_rate_ =" << max_frame_rate_ << "}) "; 143 } 144 145 VideoTrackAdapter:: 146 VideoFrameResolutionAdapter::~VideoFrameResolutionAdapter() { 147 DCHECK(io_thread_checker_.CalledOnValidThread()); 148 DCHECK(callbacks_.empty()); 149 } 150 151 void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame( 152 const scoped_refptr<media::VideoFrame>& frame, 153 const media::VideoCaptureFormat& format, 154 const base::TimeTicks& estimated_capture_time) { 155 DCHECK(io_thread_checker_.CalledOnValidThread()); 156 157 if (MaybeDropFrame(frame, format.frame_rate)) 158 return; 159 160 // TODO(perkj): Allow cropping / scaling of textures once 161 // http://crbug/362521 is fixed. 162 if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) { 163 DoDeliverFrame(frame, format, estimated_capture_time); 164 return; 165 } 166 scoped_refptr<media::VideoFrame> video_frame(frame); 167 double input_ratio = 168 static_cast<double>(frame->natural_size().width()) / 169 frame->natural_size().height(); 170 171 // If |frame| has larger width or height than requested, or the aspect ratio 172 // does not match the requested, we want to create a wrapped version of this 173 // frame with a size that fulfills the constraints. 174 if (frame->natural_size().width() > max_frame_size_.width() || 175 frame->natural_size().height() > max_frame_size_.height() || 176 input_ratio > max_aspect_ratio_ || 177 input_ratio < min_aspect_ratio_) { 178 int desired_width = std::min(max_frame_size_.width(), 179 frame->natural_size().width()); 180 int desired_height = std::min(max_frame_size_.height(), 181 frame->natural_size().height()); 182 183 double resulting_ratio = 184 static_cast<double>(desired_width) / desired_height; 185 double requested_ratio = resulting_ratio; 186 187 if (requested_ratio > max_aspect_ratio_) 188 requested_ratio = max_aspect_ratio_; 189 else if (requested_ratio < min_aspect_ratio_) 190 requested_ratio = min_aspect_ratio_; 191 192 if (resulting_ratio < requested_ratio) { 193 desired_height = static_cast<int>((desired_height * resulting_ratio) / 194 requested_ratio); 195 // Make sure we scale to an even height to avoid rounding errors 196 desired_height = (desired_height + 1) & ~1; 197 } else if (resulting_ratio > requested_ratio) { 198 desired_width = static_cast<int>((desired_width * requested_ratio) / 199 resulting_ratio); 200 // Make sure we scale to an even width to avoid rounding errors. 201 desired_width = (desired_width + 1) & ~1; 202 } 203 204 gfx::Size desired_size(desired_width, desired_height); 205 206 // Get the largest centered rectangle with the same aspect ratio of 207 // |desired_size| that fits entirely inside of |frame->visible_rect()|. 208 // This will be the rect we need to crop the original frame to. 209 // From this rect, the original frame can be scaled down to |desired_size|. 210 gfx::Rect region_in_frame = 211 media::ComputeLetterboxRegion(frame->visible_rect(), desired_size); 212 213 video_frame = media::VideoFrame::WrapVideoFrame( 214 frame, 215 region_in_frame, 216 desired_size, 217 base::Bind(&ReleaseOriginalFrame, frame)); 218 219 DVLOG(3) << "desired size " << desired_size.ToString() 220 << " output natural size " 221 << video_frame->natural_size().ToString() 222 << " output visible rect " 223 << video_frame->visible_rect().ToString(); 224 } 225 DoDeliverFrame(video_frame, format, estimated_capture_time); 226 } 227 228 bool VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeDropFrame( 229 const scoped_refptr<media::VideoFrame>& frame, 230 float source_frame_rate) { 231 DCHECK(io_thread_checker_.CalledOnValidThread()); 232 233 // Do not drop frames if max frame rate hasn't been specified or the source 234 // frame rate is known and is lower than max. 235 if (max_frame_rate_ == 0.0f || 236 (source_frame_rate > 0 && 237 source_frame_rate <= max_frame_rate_)) { 238 return false; 239 } 240 241 base::TimeDelta delta = frame->timestamp() - last_time_stamp_; 242 if (delta.InMilliseconds() < kMinTimeInMsBetweenFrames) { 243 // We have seen video frames being delivered from camera devices back to 244 // back. The simple AR filter for frame rate calculation is too short to 245 // handle that. http://crbug/394315 246 // TODO(perkj): Can we come up with a way to fix the times stamps and the 247 // timing when frames are delivered so all frames can be used? 248 // The time stamps are generated by Chrome and not the actual device. 249 // Most likely the back to back problem is caused by software and not the 250 // actual camera. 251 DVLOG(3) << "Drop frame since delta time since previous frame is " 252 << delta.InMilliseconds() << "ms."; 253 return true; 254 } 255 last_time_stamp_ = frame->timestamp(); 256 if (delta == last_time_stamp_) // First received frame. 257 return false; 258 // Calculate the frame rate using a simple AR filter. 259 // Use a simple filter with 0.1 weight of the current sample. 260 frame_rate_ = 100 / delta.InMillisecondsF() + 0.9 * frame_rate_; 261 262 // Prefer to not drop frames. 263 if (max_frame_rate_ + 0.5f > frame_rate_) 264 return false; // Keep this frame. 265 266 // The input frame rate is higher than requested. 267 // Decide if we should keep this frame or drop it. 268 keep_frame_counter_ += max_frame_rate_ / frame_rate_; 269 if (keep_frame_counter_ >= 1) { 270 keep_frame_counter_ -= 1; 271 // Keep the frame. 272 return false; 273 } 274 DVLOG(3) << "Drop frame. Input frame_rate_ " << frame_rate_ << "."; 275 return true; 276 } 277 278 void VideoTrackAdapter:: 279 VideoFrameResolutionAdapter::DoDeliverFrame( 280 const scoped_refptr<media::VideoFrame>& frame, 281 const media::VideoCaptureFormat& format, 282 const base::TimeTicks& estimated_capture_time) { 283 DCHECK(io_thread_checker_.CalledOnValidThread()); 284 for (std::vector<VideoIdCallbackPair>::const_iterator it = callbacks_.begin(); 285 it != callbacks_.end(); ++it) { 286 it->second.Run(frame, format, estimated_capture_time); 287 } 288 } 289 290 void VideoTrackAdapter::VideoFrameResolutionAdapter::AddCallback( 291 const MediaStreamVideoTrack* track, 292 const VideoCaptureDeliverFrameCB& callback) { 293 DCHECK(io_thread_checker_.CalledOnValidThread()); 294 callbacks_.push_back(std::make_pair(track, callback)); 295 } 296 297 void VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveCallback( 298 const MediaStreamVideoTrack* track) { 299 DCHECK(io_thread_checker_.CalledOnValidThread()); 300 std::vector<VideoIdCallbackPair>::iterator it = callbacks_.begin(); 301 for (; it != callbacks_.end(); ++it) { 302 if (it->first == track) { 303 // Make sure the VideoCaptureDeliverFrameCB is released on the main 304 // render thread since it was added on the main render thread in 305 // VideoTrackAdapter::AddTrack. 306 scoped_ptr<VideoCaptureDeliverFrameCB> callback( 307 new VideoCaptureDeliverFrameCB(it->second)); 308 callbacks_.erase(it); 309 renderer_task_runner_->PostTask( 310 FROM_HERE, base::Bind(&ResetCallbackOnMainRenderThread, 311 base::Passed(&callback))); 312 313 return; 314 } 315 } 316 } 317 318 bool VideoTrackAdapter::VideoFrameResolutionAdapter::ConstraintsMatch( 319 const gfx::Size& max_size, 320 double min_aspect_ratio, 321 double max_aspect_ratio, 322 double max_frame_rate) const { 323 DCHECK(io_thread_checker_.CalledOnValidThread()); 324 return max_frame_size_ == max_size && 325 min_aspect_ratio_ == min_aspect_ratio && 326 max_aspect_ratio_ == max_aspect_ratio && 327 max_frame_rate_ == max_frame_rate; 328 } 329 330 bool VideoTrackAdapter::VideoFrameResolutionAdapter::IsEmpty() const { 331 DCHECK(io_thread_checker_.CalledOnValidThread()); 332 return callbacks_.empty(); 333 } 334 335 VideoTrackAdapter::VideoTrackAdapter( 336 const scoped_refptr<base::MessageLoopProxy>& io_message_loop) 337 : io_message_loop_(io_message_loop), 338 renderer_task_runner_(base::MessageLoopProxy::current()), 339 monitoring_frame_rate_(false), 340 muted_state_(false), 341 frame_counter_(0), 342 source_frame_rate_(0.0f) { 343 DCHECK(io_message_loop_.get()); 344 } 345 346 VideoTrackAdapter::~VideoTrackAdapter() { 347 DCHECK(adapters_.empty()); 348 } 349 350 void VideoTrackAdapter::AddTrack( 351 const MediaStreamVideoTrack* track, 352 VideoCaptureDeliverFrameCB frame_callback, 353 int max_width, 354 int max_height, 355 double min_aspect_ratio, 356 double max_aspect_ratio, 357 double max_frame_rate) { 358 DCHECK(thread_checker_.CalledOnValidThread()); 359 360 io_message_loop_->PostTask( 361 FROM_HERE, 362 base::Bind(&VideoTrackAdapter::AddTrackOnIO, 363 this, track, frame_callback, gfx::Size(max_width, max_height), 364 min_aspect_ratio, max_aspect_ratio, max_frame_rate)); 365 } 366 367 void VideoTrackAdapter::AddTrackOnIO( 368 const MediaStreamVideoTrack* track, 369 VideoCaptureDeliverFrameCB frame_callback, 370 const gfx::Size& max_frame_size, 371 double min_aspect_ratio, 372 double max_aspect_ratio, 373 double max_frame_rate) { 374 DCHECK(io_message_loop_->BelongsToCurrentThread()); 375 scoped_refptr<VideoFrameResolutionAdapter> adapter; 376 for (FrameAdapters::const_iterator it = adapters_.begin(); 377 it != adapters_.end(); ++it) { 378 if ((*it)->ConstraintsMatch(max_frame_size, min_aspect_ratio, 379 max_aspect_ratio, max_frame_rate)) { 380 adapter = it->get(); 381 break; 382 } 383 } 384 if (!adapter.get()) { 385 adapter = new VideoFrameResolutionAdapter(renderer_task_runner_, 386 max_frame_size, 387 min_aspect_ratio, 388 max_aspect_ratio, 389 max_frame_rate); 390 adapters_.push_back(adapter); 391 } 392 393 adapter->AddCallback(track, frame_callback); 394 } 395 396 void VideoTrackAdapter::RemoveTrack(const MediaStreamVideoTrack* track) { 397 DCHECK(thread_checker_.CalledOnValidThread()); 398 io_message_loop_->PostTask( 399 FROM_HERE, 400 base::Bind(&VideoTrackAdapter::RemoveTrackOnIO, this, track)); 401 } 402 403 void VideoTrackAdapter::StartFrameMonitoring( 404 double source_frame_rate, 405 const OnMutedCallback& on_muted_callback) { 406 DCHECK(thread_checker_.CalledOnValidThread()); 407 408 VideoTrackAdapter::OnMutedCallback bound_on_muted_callback = 409 media::BindToCurrentLoop(on_muted_callback); 410 411 io_message_loop_->PostTask( 412 FROM_HERE, 413 base::Bind(&VideoTrackAdapter::StartFrameMonitoringOnIO, 414 this, bound_on_muted_callback, source_frame_rate)); 415 } 416 417 void VideoTrackAdapter::StartFrameMonitoringOnIO( 418 const OnMutedCallback& on_muted_callback, 419 double source_frame_rate) { 420 DCHECK(io_message_loop_->BelongsToCurrentThread()); 421 DCHECK(!monitoring_frame_rate_); 422 423 monitoring_frame_rate_ = true; 424 425 // If the source does not know the frame rate, set one by default. 426 if (source_frame_rate == 0.0f) 427 source_frame_rate = MediaStreamVideoSource::kDefaultFrameRate; 428 source_frame_rate_ = source_frame_rate; 429 DVLOG(1) << "Monitoring frame creation, first (large) delay: " 430 << (kFirstFrameTimeoutInFrameIntervals / source_frame_rate_) << "s"; 431 io_message_loop_->PostDelayedTask(FROM_HERE, 432 base::Bind(&VideoTrackAdapter::CheckFramesReceivedOnIO, this, 433 on_muted_callback, frame_counter_), 434 base::TimeDelta::FromSecondsD(kFirstFrameTimeoutInFrameIntervals / 435 source_frame_rate_)); 436 } 437 438 void VideoTrackAdapter::StopFrameMonitoring() { 439 DCHECK(thread_checker_.CalledOnValidThread()); 440 io_message_loop_->PostTask( 441 FROM_HERE, 442 base::Bind(&VideoTrackAdapter::StopFrameMonitoringOnIO, this)); 443 } 444 445 void VideoTrackAdapter::StopFrameMonitoringOnIO() { 446 DCHECK(io_message_loop_->BelongsToCurrentThread()); 447 monitoring_frame_rate_ = false; 448 } 449 450 void VideoTrackAdapter::RemoveTrackOnIO(const MediaStreamVideoTrack* track) { 451 DCHECK(io_message_loop_->BelongsToCurrentThread()); 452 for (FrameAdapters::iterator it = adapters_.begin(); 453 it != adapters_.end(); ++it) { 454 (*it)->RemoveCallback(track); 455 if ((*it)->IsEmpty()) { 456 adapters_.erase(it); 457 break; 458 } 459 } 460 } 461 462 void VideoTrackAdapter::DeliverFrameOnIO( 463 const scoped_refptr<media::VideoFrame>& frame, 464 const media::VideoCaptureFormat& format, 465 const base::TimeTicks& estimated_capture_time) { 466 DCHECK(io_message_loop_->BelongsToCurrentThread()); 467 TRACE_EVENT0("video", "VideoTrackAdapter::DeliverFrameOnIO"); 468 ++frame_counter_; 469 for (FrameAdapters::iterator it = adapters_.begin(); 470 it != adapters_.end(); ++it) { 471 (*it)->DeliverFrame(frame, format, estimated_capture_time); 472 } 473 } 474 475 void VideoTrackAdapter::CheckFramesReceivedOnIO( 476 const OnMutedCallback& set_muted_state_callback, 477 uint64 old_frame_counter_snapshot) { 478 DCHECK(io_message_loop_->BelongsToCurrentThread()); 479 480 if (!monitoring_frame_rate_) 481 return; 482 483 DVLOG_IF(1, old_frame_counter_snapshot == frame_counter_) 484 << "No frames have passed, setting source as Muted."; 485 486 bool muted_state = old_frame_counter_snapshot == frame_counter_; 487 if (muted_state_ != muted_state) { 488 set_muted_state_callback.Run(muted_state); 489 muted_state_ = muted_state; 490 } 491 492 io_message_loop_->PostDelayedTask(FROM_HERE, 493 base::Bind(&VideoTrackAdapter::CheckFramesReceivedOnIO, this, 494 set_muted_state_callback, frame_counter_), 495 base::TimeDelta::FromSecondsD(kNormalFrameTimeoutInFrameIntervals / 496 source_frame_rate_)); 497 } 498 499 } // namespace content 500
