Home | History | Annotate | Download | only in capture 
      1 // Copyright 2014 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "content/browser/media/capture/content_video_capture_device_core.h"
      6 
      7 #include "base/basictypes.h"
      8 #include "base/bind.h"
      9 #include "base/callback_forward.h"
     10 #include "base/callback_helpers.h"
     11 #include "base/debug/trace_event.h"
     12 #include "base/logging.h"
     13 #include "base/memory/scoped_ptr.h"
     14 #include "base/memory/weak_ptr.h"
     15 #include "base/message_loop/message_loop_proxy.h"
     16 #include "base/metrics/histogram.h"
     17 #include "base/sequenced_task_runner.h"
     18 #include "base/strings/string_number_conversions.h"
     19 #include "base/strings/stringprintf.h"
     20 #include "base/synchronization/lock.h"
     21 #include "base/threading/thread.h"
     22 #include "base/threading/thread_checker.h"
     23 #include "base/time/time.h"
     24 #include "content/public/browser/browser_thread.h"
     25 #include "media/base/bind_to_current_loop.h"
     26 #include "media/base/video_frame.h"
     27 #include "media/base/video_util.h"
     28 #include "media/video/capture/video_capture_types.h"
     29 #include "ui/gfx/rect.h"
     30 
     31 namespace content {
     32 
     33 namespace {
     34 
     35 void DeleteCaptureMachineOnUIThread(
     36     scoped_ptr<VideoCaptureMachine> capture_machine) {
     37   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
     38 
     39   capture_machine.reset();
     40 }
     41 
     42 }  // namespace
     43 
     44 ThreadSafeCaptureOracle::ThreadSafeCaptureOracle(
     45     scoped_ptr<media::VideoCaptureDevice::Client> client,
     46     scoped_ptr<VideoCaptureOracle> oracle,
     47     const media::VideoCaptureParams& params)
     48     : client_(client.Pass()),
     49       oracle_(oracle.Pass()),
     50       params_(params),
     51       capture_size_updated_(false) {
     52   switch (params_.requested_format.pixel_format) {
     53     case media::PIXEL_FORMAT_I420:
     54       video_frame_format_ = media::VideoFrame::I420;
     55       break;
     56     case media::PIXEL_FORMAT_TEXTURE:
     57       video_frame_format_ = media::VideoFrame::NATIVE_TEXTURE;
     58       break;
     59     default:
     60       LOG(FATAL) << "Unexpected pixel_format "
     61                  << params_.requested_format.pixel_format;
     62   }
     63 }
     64 
     65 ThreadSafeCaptureOracle::~ThreadSafeCaptureOracle() {}
     66 
     67 bool ThreadSafeCaptureOracle::ObserveEventAndDecideCapture(
     68     VideoCaptureOracle::Event event,
     69     const gfx::Rect& damage_rect,
     70     base::TimeTicks event_time,
     71     scoped_refptr<media::VideoFrame>* storage,
     72     CaptureFrameCallback* callback) {
     73   base::AutoLock guard(lock_);
     74 
     75   if (!client_)
     76     return false;  // Capture is stopped.
     77 
     78   // Always round up the coded size to multiple of 16 pixels.
     79   // See http://crbug.com/402151.
     80   const gfx::Size visible_size = params_.requested_format.frame_size;
     81   const gfx::Size coded_size((visible_size.width() + 15) & ~15,
     82                              (visible_size.height() + 15) & ~15);
     83 
     84   scoped_refptr<media::VideoCaptureDevice::Client::Buffer> output_buffer =
     85       client_->ReserveOutputBuffer(video_frame_format_, coded_size);
     86   const bool should_capture =
     87       oracle_->ObserveEventAndDecideCapture(event, damage_rect, event_time);
     88   const bool content_is_dirty =
     89       (event == VideoCaptureOracle::kCompositorUpdate ||
     90        event == VideoCaptureOracle::kSoftwarePaint);
     91   const char* event_name =
     92       (event == VideoCaptureOracle::kTimerPoll ? "poll" :
     93        (event == VideoCaptureOracle::kCompositorUpdate ? "gpu" :
     94        "paint"));
     95 
     96   // Consider the various reasons not to initiate a capture.
     97   if (should_capture && !output_buffer.get()) {
     98     TRACE_EVENT_INSTANT1("mirroring",
     99                          "PipelineLimited",
    100                          TRACE_EVENT_SCOPE_THREAD,
    101                          "trigger",
    102                          event_name);
    103     return false;
    104   } else if (!should_capture && output_buffer.get()) {
    105     if (content_is_dirty) {
    106       // This is a normal and acceptable way to drop a frame. We've hit our
    107       // capture rate limit: for example, the content is animating at 60fps but
    108       // we're capturing at 30fps.
    109       TRACE_EVENT_INSTANT1("mirroring", "FpsRateLimited",
    110                            TRACE_EVENT_SCOPE_THREAD,
    111                            "trigger", event_name);
    112     }
    113     return false;
    114   } else if (!should_capture && !output_buffer.get()) {
    115     // We decided not to capture, but we wouldn't have been able to if we wanted
    116     // to because no output buffer was available.
    117     TRACE_EVENT_INSTANT1("mirroring", "NearlyPipelineLimited",
    118                          TRACE_EVENT_SCOPE_THREAD,
    119                          "trigger", event_name);
    120     return false;
    121   }
    122   int frame_number = oracle_->RecordCapture();
    123   TRACE_EVENT_ASYNC_BEGIN2("mirroring", "Capture", output_buffer.get(),
    124                            "frame_number", frame_number,
    125                            "trigger", event_name);
    126   // NATIVE_TEXTURE frames wrap a texture mailbox, which we don't have at the
    127   // moment.  We do not construct those frames.
    128   if (video_frame_format_ != media::VideoFrame::NATIVE_TEXTURE) {
    129     *storage = media::VideoFrame::WrapExternalPackedMemory(
    130         video_frame_format_,
    131         coded_size,
    132         gfx::Rect(visible_size),
    133         visible_size,
    134         static_cast<uint8*>(output_buffer->data()),
    135         output_buffer->size(),
    136         base::SharedMemory::NULLHandle(),
    137         base::TimeDelta(),
    138         base::Closure());
    139   }
    140   *callback = base::Bind(&ThreadSafeCaptureOracle::DidCaptureFrame,
    141                          this,
    142                          frame_number,
    143                          output_buffer);
    144   return true;
    145 }
    146 
    147 gfx::Size ThreadSafeCaptureOracle::GetCaptureSize() const {
    148   base::AutoLock guard(lock_);
    149   return params_.requested_format.frame_size;
    150 }
    151 
    152 void ThreadSafeCaptureOracle::UpdateCaptureSize(const gfx::Size& source_size) {
    153   base::AutoLock guard(lock_);
    154 
    155   // If this is the first call to UpdateCaptureSize(), or the receiver supports
    156   // variable resolution, then determine the capture size by treating the
    157   // requested width and height as maxima.
    158   if (!capture_size_updated_ || params_.resolution_change_policy ==
    159       media::RESOLUTION_POLICY_DYNAMIC_WITHIN_LIMIT) {
    160     // The capture resolution should not exceed the source frame size.
    161     // In other words it should downscale the image but not upscale it.
    162     if (source_size.width() > params_.requested_format.frame_size.width() ||
    163         source_size.height() > params_.requested_format.frame_size.height()) {
    164       gfx::Rect capture_rect = media::ComputeLetterboxRegion(
    165           gfx::Rect(params_.requested_format.frame_size), source_size);
    166       params_.requested_format.frame_size.SetSize(
    167           MakeEven(capture_rect.width()), MakeEven(capture_rect.height()));
    168     } else {
    169       params_.requested_format.frame_size.SetSize(
    170           MakeEven(source_size.width()), MakeEven(source_size.height()));
    171     }
    172     capture_size_updated_ = true;
    173   }
    174 }
    175 
    176 void ThreadSafeCaptureOracle::Stop() {
    177   base::AutoLock guard(lock_);
    178   client_.reset();
    179 }
    180 
    181 void ThreadSafeCaptureOracle::ReportError(const std::string& reason) {
    182   base::AutoLock guard(lock_);
    183   if (client_)
    184     client_->OnError(reason);
    185 }
    186 
    187 void ThreadSafeCaptureOracle::DidCaptureFrame(
    188     int frame_number,
    189     const scoped_refptr<media::VideoCaptureDevice::Client::Buffer>& buffer,
    190     const scoped_refptr<media::VideoFrame>& frame,
    191     base::TimeTicks timestamp,
    192     bool success) {
    193   base::AutoLock guard(lock_);
    194   TRACE_EVENT_ASYNC_END2("mirroring", "Capture", buffer.get(),
    195                          "success", success,
    196                          "timestamp", timestamp.ToInternalValue());
    197 
    198   if (!client_)
    199     return;  // Capture is stopped.
    200 
    201   if (success) {
    202     if (oracle_->CompleteCapture(frame_number, &timestamp)) {
    203       media::VideoCaptureFormat format = params_.requested_format;
    204       format.frame_size = frame->coded_size();
    205       client_->OnIncomingCapturedVideoFrame(buffer, format, frame, timestamp);
    206     }
    207   }
    208 }
    209 
    210 void ContentVideoCaptureDeviceCore::AllocateAndStart(
    211     const media::VideoCaptureParams& params,
    212     scoped_ptr<media::VideoCaptureDevice::Client> client) {
    213   DCHECK(thread_checker_.CalledOnValidThread());
    214 
    215   if (state_ != kIdle) {
    216     DVLOG(1) << "Allocate() invoked when not in state Idle.";
    217     return;
    218   }
    219 
    220   if (params.requested_format.frame_rate <= 0) {
    221     std::string error_msg("Invalid frame_rate: ");
    222     error_msg += base::DoubleToString(params.requested_format.frame_rate);
    223     DVLOG(1) << error_msg;
    224     client->OnError(error_msg);
    225     return;
    226   }
    227 
    228   if (params.requested_format.pixel_format != media::PIXEL_FORMAT_I420 &&
    229       params.requested_format.pixel_format != media::PIXEL_FORMAT_TEXTURE) {
    230     std::string error_msg = base::StringPrintf(
    231         "unsupported format: %d", params.requested_format.pixel_format);
    232     DVLOG(1) << error_msg;
    233     client->OnError(error_msg);
    234     return;
    235   }
    236 
    237    if (params.requested_format.frame_size.width() < kMinFrameWidth ||
    238        params.requested_format.frame_size.height() < kMinFrameHeight) {
    239      std::string error_msg =
    240          "invalid frame size: " + params.requested_format.frame_size.ToString();
    241      DVLOG(1) << error_msg;
    242      client->OnError(error_msg);
    243      return;
    244    }
    245 
    246   media::VideoCaptureParams new_params = params;
    247   // Frame dimensions must each be an even integer since the client wants (or
    248   // will convert to) YUV420.
    249   new_params.requested_format.frame_size.SetSize(
    250       MakeEven(params.requested_format.frame_size.width()),
    251       MakeEven(params.requested_format.frame_size.height()));
    252 
    253   base::TimeDelta capture_period = base::TimeDelta::FromMicroseconds(
    254       1000000.0 / params.requested_format.frame_rate + 0.5);
    255 
    256   scoped_ptr<VideoCaptureOracle> oracle(
    257       new VideoCaptureOracle(capture_period,
    258                              kAcceleratedSubscriberIsSupported));
    259   oracle_proxy_ =
    260       new ThreadSafeCaptureOracle(client.Pass(), oracle.Pass(), new_params);
    261 
    262   // Starts the capture machine asynchronously.
    263   BrowserThread::PostTaskAndReplyWithResult(
    264       BrowserThread::UI,
    265       FROM_HERE,
    266       base::Bind(&VideoCaptureMachine::Start,
    267                  base::Unretained(capture_machine_.get()),
    268                  oracle_proxy_,
    269                  new_params),
    270       base::Bind(&ContentVideoCaptureDeviceCore::CaptureStarted, AsWeakPtr()));
    271 
    272   TransitionStateTo(kCapturing);
    273 }
    274 
    275 void ContentVideoCaptureDeviceCore::StopAndDeAllocate() {
    276   DCHECK(thread_checker_.CalledOnValidThread());
    277 
    278   if (state_ != kCapturing)
    279     return;
    280 
    281   oracle_proxy_->Stop();
    282   oracle_proxy_ = NULL;
    283 
    284   TransitionStateTo(kIdle);
    285 
    286   // Stops the capture machine asynchronously.
    287   BrowserThread::PostTask(
    288       BrowserThread::UI, FROM_HERE, base::Bind(
    289           &VideoCaptureMachine::Stop,
    290           base::Unretained(capture_machine_.get()),
    291           base::Bind(&base::DoNothing)));
    292 }
    293 
    294 void ContentVideoCaptureDeviceCore::CaptureStarted(bool success) {
    295   DCHECK(thread_checker_.CalledOnValidThread());
    296   if (!success) {
    297     std::string reason("Failed to start capture machine.");
    298     DVLOG(1) << reason;
    299     Error(reason);
    300   }
    301 }
    302 
    303 ContentVideoCaptureDeviceCore::ContentVideoCaptureDeviceCore(
    304     scoped_ptr<VideoCaptureMachine> capture_machine)
    305     : state_(kIdle),
    306       capture_machine_(capture_machine.Pass()) {
    307   DCHECK(capture_machine_.get());
    308 }
    309 
    310 ContentVideoCaptureDeviceCore::~ContentVideoCaptureDeviceCore() {
    311   DCHECK(thread_checker_.CalledOnValidThread());
    312   DCHECK_NE(state_, kCapturing);
    313   // If capture_machine is not NULL, then we need to return to the UI thread to
    314   // safely stop the capture machine.
    315   if (capture_machine_) {
    316     VideoCaptureMachine* capture_machine_ptr = capture_machine_.get();
    317     BrowserThread::PostTask(
    318         BrowserThread::UI, FROM_HERE,
    319         base::Bind(&VideoCaptureMachine::Stop,
    320                    base::Unretained(capture_machine_ptr),
    321                    base::Bind(&DeleteCaptureMachineOnUIThread,
    322                               base::Passed(&capture_machine_))));
    323   }
    324   DVLOG(1) << "ContentVideoCaptureDeviceCore@" << this << " destroying.";
    325 }
    326 
    327 void ContentVideoCaptureDeviceCore::TransitionStateTo(State next_state) {
    328   DCHECK(thread_checker_.CalledOnValidThread());
    329 
    330 #ifndef NDEBUG
    331   static const char* kStateNames[] = {
    332     "Idle", "Allocated", "Capturing", "Error"
    333   };
    334   DVLOG(1) << "State change: " << kStateNames[state_]
    335            << " --> " << kStateNames[next_state];
    336 #endif
    337 
    338   state_ = next_state;
    339 }
    340 
    341 void ContentVideoCaptureDeviceCore::Error(const std::string& reason) {
    342   DCHECK(thread_checker_.CalledOnValidThread());
    343 
    344   if (state_ == kIdle)
    345     return;
    346 
    347   if (oracle_proxy_.get())
    348     oracle_proxy_->ReportError(reason);
    349 
    350   StopAndDeAllocate();
    351   TransitionStateTo(kError);
    352 }
    353 
    354 }  // namespace content
    355