Home | History | Annotate | Download | only in media 
      1 // Copyright 2014 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "content/renderer/media/video_track_adapter.h"
      6 
      7 #include <algorithm>
      8 #include <limits>
      9 #include <utility>
     10 
     11 #include "base/bind.h"
     12 #include "base/debug/trace_event.h"
     13 #include "base/location.h"
     14 #include "media/base/video_util.h"
     15 
     16 namespace content {
     17 
     18 namespace {
     19 
     20 // Empty method used for keeping a reference to the original media::VideoFrame
     21 // in VideoFrameResolutionAdapter::DeliverFrame if cropping is needed.
     22 // The reference to |frame| is kept in the closure that calls this method.
     23 void ReleaseOriginalFrame(
     24     const scoped_refptr<media::VideoFrame>& frame) {
     25 }
     26 
     27 void ResetCallbackOnMainRenderThread(
     28     scoped_ptr<VideoCaptureDeliverFrameCB> callback) {
     29   // |callback| will be deleted when this exits.
     30 }
     31 
     32 }  // anonymous namespace
     33 
     34 // VideoFrameResolutionAdapter is created on and lives on
     35 // on the IO-thread. It does the resolution adaptation and delivers frames to
     36 // all registered tracks on the IO-thread.
     37 // All method calls must be on the IO-thread.
     38 class VideoTrackAdapter::VideoFrameResolutionAdapter
     39     : public base::RefCountedThreadSafe<VideoFrameResolutionAdapter> {
     40  public:
     41   VideoFrameResolutionAdapter(
     42       scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
     43       int max_width,
     44       int max_height,
     45       double min_aspect_ratio,
     46       double max_aspect_ratio);
     47 
     48   // Add |callback| to receive video frames on the IO-thread.
     49   // |callback| will however be released on the main render thread.
     50   void AddCallback(const MediaStreamVideoTrack* track,
     51                    const VideoCaptureDeliverFrameCB& callback);
     52 
     53   // Removes |callback| associated with |track| from receiving video frames if
     54   // |track| has been added. It is ok to call RemoveCallback even if the |track|
     55   // has not been added. The |callback| is released on the main render thread.
     56   void RemoveCallback(const MediaStreamVideoTrack* track);
     57 
     58   void DeliverFrame(const scoped_refptr<media::VideoFrame>& frame,
     59                     const media::VideoCaptureFormat& format,
     60                     const base::TimeTicks& estimated_capture_time);
     61 
     62   // Returns true if all arguments match with the output of this adapter.
     63   bool ConstraintsMatch(int max_width,
     64                         int max_height,
     65                         double min_aspect_ratio,
     66                         double max_aspect_ratio) const;
     67 
     68   bool IsEmpty() const;
     69 
     70  private:
     71   virtual ~VideoFrameResolutionAdapter();
     72   friend class base::RefCountedThreadSafe<VideoFrameResolutionAdapter>;
     73 
     74   virtual void DoDeliverFrame(
     75       const scoped_refptr<media::VideoFrame>& frame,
     76       const media::VideoCaptureFormat& format,
     77       const base::TimeTicks& estimated_capture_time);
     78 
     79   // Bound to the IO-thread.
     80   base::ThreadChecker io_thread_checker_;
     81 
     82   // The task runner where we will release VideoCaptureDeliverFrameCB
     83   // registered in AddCallback.
     84   scoped_refptr<base::SingleThreadTaskRunner> renderer_task_runner_;
     85 
     86   gfx::Size max_frame_size_;
     87   double min_aspect_ratio_;
     88   double max_aspect_ratio_;
     89 
     90   typedef std::pair<const void*, VideoCaptureDeliverFrameCB>
     91       VideoIdCallbackPair;
     92   std::vector<VideoIdCallbackPair> callbacks_;
     93 
     94   DISALLOW_COPY_AND_ASSIGN(VideoFrameResolutionAdapter);
     95 };
     96 
     97 VideoTrackAdapter::
     98 VideoFrameResolutionAdapter::VideoFrameResolutionAdapter(
     99     scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
    100     int max_width,
    101     int max_height,
    102     double min_aspect_ratio,
    103     double max_aspect_ratio)
    104     : renderer_task_runner_(render_message_loop),
    105       max_frame_size_(max_width, max_height),
    106       min_aspect_ratio_(min_aspect_ratio),
    107       max_aspect_ratio_(max_aspect_ratio) {
    108   DCHECK(renderer_task_runner_);
    109   DCHECK(io_thread_checker_.CalledOnValidThread());
    110   DCHECK_GE(max_aspect_ratio_, min_aspect_ratio_);
    111   CHECK_NE(0, max_aspect_ratio_);
    112   DVLOG(3) << "VideoFrameResolutionAdapter("
    113           << "{ max_width =" << max_width << "}, "
    114           << "{ max_height =" << max_height << "}, "
    115           << "{ min_aspect_ratio =" << min_aspect_ratio << "}, "
    116           << "{ max_aspect_ratio_ =" << max_aspect_ratio_ << "}) ";
    117 }
    118 
    119 VideoTrackAdapter::
    120 VideoFrameResolutionAdapter::~VideoFrameResolutionAdapter() {
    121   DCHECK(io_thread_checker_.CalledOnValidThread());
    122   DCHECK(callbacks_.empty());
    123 }
    124 
    125 void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame(
    126     const scoped_refptr<media::VideoFrame>& frame,
    127     const media::VideoCaptureFormat& format,
    128     const base::TimeTicks& estimated_capture_time) {
    129   DCHECK(io_thread_checker_.CalledOnValidThread());
    130   // TODO(perkj): Allow cropping / scaling of textures once
    131   // http://crbug/362521 is fixed.
    132   if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
    133     DoDeliverFrame(frame, format, estimated_capture_time);
    134     return;
    135   }
    136   scoped_refptr<media::VideoFrame> video_frame(frame);
    137   double input_ratio =
    138       static_cast<double>(frame->natural_size().width()) /
    139       frame->natural_size().height();
    140 
    141   // If |frame| has larger width or height than requested, or the aspect ratio
    142   // does not match the requested, we want to create a wrapped version of this
    143   // frame with a size that fulfills the constraints.
    144   if (frame->natural_size().width() > max_frame_size_.width() ||
    145       frame->natural_size().height() > max_frame_size_.height() ||
    146       input_ratio > max_aspect_ratio_ ||
    147       input_ratio < min_aspect_ratio_) {
    148     int desired_width = std::min(max_frame_size_.width(),
    149                                  frame->natural_size().width());
    150     int desired_height = std::min(max_frame_size_.height(),
    151                                   frame->natural_size().height());
    152 
    153     double resulting_ratio =
    154         static_cast<double>(desired_width) / desired_height;
    155     double requested_ratio = resulting_ratio;
    156 
    157     if (requested_ratio > max_aspect_ratio_)
    158       requested_ratio = max_aspect_ratio_;
    159     else if (requested_ratio < min_aspect_ratio_)
    160       requested_ratio = min_aspect_ratio_;
    161 
    162     if (resulting_ratio < requested_ratio) {
    163       desired_height = static_cast<int>((desired_height * resulting_ratio) /
    164                                         requested_ratio);
    165       // Make sure we scale to an even height to avoid rounding errors
    166       desired_height = (desired_height + 1) & ~1;
    167     } else if (resulting_ratio > requested_ratio) {
    168       desired_width = static_cast<int>((desired_width * requested_ratio) /
    169                                        resulting_ratio);
    170       // Make sure we scale to an even width to avoid rounding errors.
    171       desired_width = (desired_width + 1) & ~1;
    172     }
    173 
    174     gfx::Size desired_size(desired_width, desired_height);
    175 
    176     // Get the largest centered rectangle with the same aspect ratio of
    177     // |desired_size| that fits entirely inside of |frame->visible_rect()|.
    178     // This will be the rect we need to crop the original frame to.
    179     // From this rect, the original frame can be scaled down to |desired_size|.
    180     gfx::Rect region_in_frame =
    181         media::ComputeLetterboxRegion(frame->visible_rect(), desired_size);
    182 
    183     video_frame = media::VideoFrame::WrapVideoFrame(
    184         frame,
    185         region_in_frame,
    186         desired_size,
    187         base::Bind(&ReleaseOriginalFrame, frame));
    188 
    189     DVLOG(3) << "desired size  " << desired_size.ToString()
    190              << " output natural size "
    191              << video_frame->natural_size().ToString()
    192              << " output visible rect  "
    193              << video_frame->visible_rect().ToString();
    194   }
    195   DoDeliverFrame(video_frame, format, estimated_capture_time);
    196 }
    197 
    198 void VideoTrackAdapter::
    199 VideoFrameResolutionAdapter::DoDeliverFrame(
    200     const scoped_refptr<media::VideoFrame>& frame,
    201     const media::VideoCaptureFormat& format,
    202     const base::TimeTicks& estimated_capture_time) {
    203   DCHECK(io_thread_checker_.CalledOnValidThread());
    204   for (std::vector<VideoIdCallbackPair>::const_iterator it = callbacks_.begin();
    205        it != callbacks_.end(); ++it) {
    206     it->second.Run(frame, format, estimated_capture_time);
    207   }
    208 }
    209 
    210 void VideoTrackAdapter::VideoFrameResolutionAdapter::AddCallback(
    211     const MediaStreamVideoTrack* track,
    212     const VideoCaptureDeliverFrameCB& callback) {
    213   DCHECK(io_thread_checker_.CalledOnValidThread());
    214   callbacks_.push_back(std::make_pair(track, callback));
    215 }
    216 
    217 void VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveCallback(
    218     const MediaStreamVideoTrack* track) {
    219   DCHECK(io_thread_checker_.CalledOnValidThread());
    220   std::vector<VideoIdCallbackPair>::iterator it = callbacks_.begin();
    221   for (; it != callbacks_.end(); ++it) {
    222     if (it->first == track) {
    223       // Make sure the VideoCaptureDeliverFrameCB is released on the main
    224       // render thread since it was added on the main render thread in
    225       // VideoTrackAdapter::AddTrack.
    226       scoped_ptr<VideoCaptureDeliverFrameCB> callback(
    227           new VideoCaptureDeliverFrameCB(it->second));
    228       callbacks_.erase(it);
    229       renderer_task_runner_->PostTask(
    230           FROM_HERE, base::Bind(&ResetCallbackOnMainRenderThread,
    231                                 base::Passed(&callback)));
    232 
    233       return;
    234     }
    235   }
    236 }
    237 
    238 bool VideoTrackAdapter::VideoFrameResolutionAdapter::ConstraintsMatch(
    239     int max_width,
    240     int max_height,
    241     double min_aspect_ratio,
    242     double max_aspect_ratio) const {
    243   DCHECK(io_thread_checker_.CalledOnValidThread());
    244   return max_frame_size_.width() == max_width &&
    245       max_frame_size_.height() == max_height &&
    246       min_aspect_ratio_ == min_aspect_ratio &&
    247       max_aspect_ratio_ == max_aspect_ratio;
    248 }
    249 
    250 bool VideoTrackAdapter::VideoFrameResolutionAdapter::IsEmpty() const {
    251   DCHECK(io_thread_checker_.CalledOnValidThread());
    252   return callbacks_.empty();
    253 }
    254 
    255 VideoTrackAdapter::VideoTrackAdapter(
    256     const scoped_refptr<base::MessageLoopProxy>& io_message_loop)
    257     : io_message_loop_(io_message_loop),
    258       renderer_task_runner_(base::MessageLoopProxy::current()) {
    259   DCHECK(io_message_loop_);
    260 }
    261 
    262 VideoTrackAdapter::~VideoTrackAdapter() {
    263   DCHECK(adapters_.empty());
    264 }
    265 
    266 void VideoTrackAdapter::AddTrack(const MediaStreamVideoTrack* track,
    267                                  VideoCaptureDeliverFrameCB frame_callback,
    268                                  int max_width,
    269                                  int max_height,
    270                                  double min_aspect_ratio,
    271                                  double max_aspect_ratio) {
    272   DCHECK(thread_checker_.CalledOnValidThread());
    273   io_message_loop_->PostTask(
    274       FROM_HERE,
    275       base::Bind(&VideoTrackAdapter::AddTrackOnIO,
    276                  this, track, frame_callback, max_width, max_height,
    277                  min_aspect_ratio, max_aspect_ratio));
    278 }
    279 
    280 void VideoTrackAdapter::AddTrackOnIO(
    281     const MediaStreamVideoTrack* track,
    282     VideoCaptureDeliverFrameCB frame_callback,
    283     int max_width,
    284     int max_height,
    285     double min_aspect_ratio,
    286     double max_aspect_ratio) {
    287   DCHECK(io_message_loop_->BelongsToCurrentThread());
    288   scoped_refptr<VideoFrameResolutionAdapter> adapter;
    289   for (FrameAdapters::const_iterator it = adapters_.begin();
    290        it != adapters_.end(); ++it) {
    291     if ((*it)->ConstraintsMatch(max_width, max_height, min_aspect_ratio,
    292                                 max_aspect_ratio)) {
    293       adapter = it->get();
    294       break;
    295     }
    296   }
    297   if (!adapter) {
    298     adapter = new VideoFrameResolutionAdapter(renderer_task_runner_,
    299                                               max_width,
    300                                               max_height,
    301                                               min_aspect_ratio,
    302                                               max_aspect_ratio);
    303     adapters_.push_back(adapter);
    304   }
    305 
    306   adapter->AddCallback(track, frame_callback);
    307 }
    308 
    309 void VideoTrackAdapter::RemoveTrack(const MediaStreamVideoTrack* track) {
    310   DCHECK(thread_checker_.CalledOnValidThread());
    311   io_message_loop_->PostTask(
    312       FROM_HERE,
    313       base::Bind(&VideoTrackAdapter::RemoveTrackOnIO, this, track));
    314 }
    315 
    316 void VideoTrackAdapter::RemoveTrackOnIO(const MediaStreamVideoTrack* track) {
    317   DCHECK(io_message_loop_->BelongsToCurrentThread());
    318   for (FrameAdapters::iterator it = adapters_.begin();
    319        it != adapters_.end(); ++it) {
    320     (*it)->RemoveCallback(track);
    321     if ((*it)->IsEmpty()) {
    322       adapters_.erase(it);
    323       break;
    324     }
    325   }
    326 }
    327 
    328 void VideoTrackAdapter::DeliverFrameOnIO(
    329     const scoped_refptr<media::VideoFrame>& frame,
    330     const media::VideoCaptureFormat& format,
    331     const base::TimeTicks& estimated_capture_time) {
    332   DCHECK(io_message_loop_->BelongsToCurrentThread());
    333   TRACE_EVENT0("video", "VideoTrackAdapter::DeliverFrameOnIO");
    334   for (FrameAdapters::iterator it = adapters_.begin();
    335        it != adapters_.end(); ++it) {
    336     (*it)->DeliverFrame(frame, format, estimated_capture_time);
    337   }
    338 }
    339 
    340 }  // namespace content
    341