1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 // 5 // Implementation notes: This needs to work on a variety of hardware 6 // configurations where the speed of the CPU and GPU greatly affect overall 7 // performance. Spanning several threads, the process of capturing has been 8 // split up into four conceptual stages: 9 // 10 // 1. Reserve Buffer: Before a frame can be captured, a slot in the client's 11 // shared-memory IPC buffer is reserved. There are only a few of these; 12 // when they run out, it indicates that the downstream client -- likely a 13 // video encoder -- is the performance bottleneck, and that the rate of 14 // frame capture should be throttled back. 15 // 16 // 2. Capture: A bitmap is snapshotted/copied from the RenderWidget's backing 17 // store. This is initiated on the UI BrowserThread, and often occurs 18 // asynchronously. Where supported, the GPU scales and color converts 19 // frames to our desired size, and the readback happens directly into the 20 // shared-memory buffer. But this is not always possible, particularly when 21 // accelerated compositing is disabled. 22 // 23 // 3. Render (if needed): If the web contents cannot be captured directly into 24 // our target size and color format, scaling and colorspace conversion must 25 // be done on the CPU. A dedicated thread is used for this operation, to 26 // avoid blocking the UI thread. The Render stage always reads from a 27 // bitmap returned by Capture, and writes into the reserved slot in the 28 // shared-memory buffer. 29 // 30 // 4. Deliver: The rendered video frame is returned to the client (which 31 // implements the VideoCaptureDevice::Client interface). Because all 32 // paths have written the frame into the IPC buffer, this step should 33 // never need to do an additional copy of the pixel data. 34 // 35 // In the best-performing case, the Render step is bypassed: Capture produces 36 // ready-to-Deliver frames. But when accelerated readback is not possible, the 37 // system is designed so that Capture and Render may run concurrently. A timing 38 // diagram helps illustrate this point (@30 FPS): 39 // 40 // Time: 0ms 33ms 66ms 99ms 41 // thread1: |-Capture-f1------v |-Capture-f2------v |-Capture-f3----v |-Capt 42 // thread2: |-Render-f1-----v |-Render-f2-----v |-Render-f3 43 // 44 // In the above example, both capturing and rendering *each* take almost the 45 // full 33 ms available between frames, yet we see that the required throughput 46 // is obtained. 47 // 48 // Turning on verbose logging will cause the effective frame rate to be logged 49 // at 5-second intervals. 50 51 #include "content/browser/media/capture/web_contents_video_capture_device.h" 52 53 #include "base/basictypes.h" 54 #include "base/bind.h" 55 #include "base/callback_helpers.h" 56 #include "base/logging.h" 57 #include "base/memory/scoped_ptr.h" 58 #include "base/memory/weak_ptr.h" 59 #include "base/message_loop/message_loop_proxy.h" 60 #include "base/metrics/histogram.h" 61 #include "base/sequenced_task_runner.h" 62 #include "base/threading/thread.h" 63 #include "base/threading/thread_checker.h" 64 #include "base/time/time.h" 65 #include "content/browser/media/capture/content_video_capture_device_core.h" 66 #include "content/browser/media/capture/video_capture_oracle.h" 67 #include "content/browser/media/capture/web_contents_capture_util.h" 68 #include "content/browser/media/capture/web_contents_tracker.h" 69 #include "content/browser/renderer_host/render_widget_host_impl.h" 70 #include "content/browser/renderer_host/render_widget_host_view_base.h" 71 #include "content/public/browser/browser_thread.h" 72 #include "content/public/browser/notification_observer.h" 73 #include "content/public/browser/notification_registrar.h" 74 #include "content/public/browser/notification_source.h" 75 #include "content/public/browser/notification_types.h" 76 #include "content/public/browser/render_process_host.h" 77 #include "content/public/browser/render_widget_host_view.h" 78 #include "content/public/browser/render_widget_host_view_frame_subscriber.h" 79 #include "content/public/browser/web_contents.h" 80 #include "media/base/video_util.h" 81 #include "media/video/capture/video_capture_types.h" 82 #include "skia/ext/image_operations.h" 83 #include "third_party/skia/include/core/SkBitmap.h" 84 #include "third_party/skia/include/core/SkColor.h" 85 #include "ui/gfx/display.h" 86 #include "ui/gfx/geometry/size.h" 87 #include "ui/gfx/geometry/size_conversions.h" 88 #include "ui/gfx/screen.h" 89 90 namespace content { 91 92 namespace { 93 94 // Compute a letterbox region, aligned to even coordinates. 95 gfx::Rect ComputeYV12LetterboxRegion(const gfx::Size& frame_size, 96 const gfx::Size& content_size) { 97 98 gfx::Rect result = media::ComputeLetterboxRegion(gfx::Rect(frame_size), 99 content_size); 100 101 result.set_x(MakeEven(result.x())); 102 result.set_y(MakeEven(result.y())); 103 result.set_width(std::max(kMinFrameWidth, MakeEven(result.width()))); 104 result.set_height(std::max(kMinFrameHeight, MakeEven(result.height()))); 105 106 return result; 107 } 108 109 void DeleteOnWorkerThread(scoped_ptr<base::Thread> render_thread, 110 const base::Closure& callback) { 111 render_thread.reset(); 112 113 // After thread join call the callback on UI thread. 114 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, callback); 115 } 116 117 // Responsible for logging the effective frame rate. 118 class VideoFrameDeliveryLog { 119 public: 120 VideoFrameDeliveryLog(); 121 122 // Report that the frame posted with |frame_time| has been delivered. 123 void ChronicleFrameDelivery(base::TimeTicks frame_time); 124 125 private: 126 // The following keep track of and log the effective frame rate whenever 127 // verbose logging is turned on. 128 base::TimeTicks last_frame_rate_log_time_; 129 int count_frames_rendered_; 130 131 DISALLOW_COPY_AND_ASSIGN(VideoFrameDeliveryLog); 132 }; 133 134 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible 135 // with RenderWidgetHostViewFrameSubscriber. We create one per event type. 136 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber { 137 public: 138 FrameSubscriber(VideoCaptureOracle::Event event_type, 139 const scoped_refptr<ThreadSafeCaptureOracle>& oracle, 140 VideoFrameDeliveryLog* delivery_log) 141 : event_type_(event_type), 142 oracle_proxy_(oracle), 143 delivery_log_(delivery_log) {} 144 145 virtual bool ShouldCaptureFrame( 146 const gfx::Rect& damage_rect, 147 base::TimeTicks present_time, 148 scoped_refptr<media::VideoFrame>* storage, 149 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback* 150 deliver_frame_cb) OVERRIDE; 151 152 private: 153 const VideoCaptureOracle::Event event_type_; 154 scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_; 155 VideoFrameDeliveryLog* const delivery_log_; 156 }; 157 158 // ContentCaptureSubscription is the relationship between a RenderWidgetHost 159 // whose content is updating, a subscriber that is deciding which of these 160 // updates to capture (and where to deliver them to), and a callback that 161 // knows how to do the capture and prepare the result for delivery. 162 // 163 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in 164 // the RenderWidgetHostView, to process updates that occur via accelerated 165 // compositing, (b) installing itself as an observer of updates to the 166 // RenderWidgetHost's backing store, to hook updates that occur via software 167 // rendering, and (c) running a timer to possibly initiate non-event-driven 168 // captures that the subscriber might request. 169 // 170 // All of this happens on the UI thread, although the 171 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates 172 // autonomously on some other thread. 173 class ContentCaptureSubscription : public content::NotificationObserver { 174 public: 175 typedef base::Callback< 176 void(const base::TimeTicks&, 177 const scoped_refptr<media::VideoFrame>&, 178 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&)> 179 CaptureCallback; 180 181 // Create a subscription. Whenever a manual capture is required, the 182 // subscription will invoke |capture_callback| on the UI thread to do the 183 // work. 184 ContentCaptureSubscription( 185 const RenderWidgetHost& source, 186 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy, 187 const CaptureCallback& capture_callback); 188 virtual ~ContentCaptureSubscription(); 189 190 // content::NotificationObserver implementation. 191 virtual void Observe(int type, 192 const content::NotificationSource& source, 193 const content::NotificationDetails& details) OVERRIDE; 194 195 private: 196 void OnTimer(); 197 198 // Maintain a weak reference to the RenderWidgetHost (via its routing ID), 199 // since the instance could be destroyed externally during the lifetime of 200 // |this|. 201 const int render_process_id_; 202 const int render_widget_id_; 203 204 VideoFrameDeliveryLog delivery_log_; 205 FrameSubscriber paint_subscriber_; 206 FrameSubscriber timer_subscriber_; 207 content::NotificationRegistrar registrar_; 208 CaptureCallback capture_callback_; 209 base::Timer timer_; 210 211 DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription); 212 }; 213 214 // Render the SkBitmap |input| into the given VideoFrame buffer |output|, then 215 // invoke |done_cb| to indicate success or failure. |input| is expected to be 216 // ARGB. |output| must be YV12 or I420. Colorspace conversion is always done. 217 // Scaling and letterboxing will be done as needed. 218 // 219 // This software implementation should be used only when GPU acceleration of 220 // these activities is not possible. This operation may be expensive (tens to 221 // hundreds of milliseconds), so the caller should ensure that it runs on a 222 // thread where such a pause would cause UI jank. 223 void RenderVideoFrame(const SkBitmap& input, 224 const scoped_refptr<media::VideoFrame>& output, 225 const base::Callback<void(bool)>& done_cb); 226 227 // Renews capture subscriptions based on feedback from WebContentsTracker, and 228 // also executes copying of the backing store on the UI BrowserThread. 229 class WebContentsCaptureMachine : public VideoCaptureMachine { 230 public: 231 WebContentsCaptureMachine(int render_process_id, int main_render_frame_id); 232 virtual ~WebContentsCaptureMachine(); 233 234 // VideoCaptureMachine overrides. 235 virtual bool Start(const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy, 236 const media::VideoCaptureParams& params) OVERRIDE; 237 virtual void Stop(const base::Closure& callback) OVERRIDE; 238 239 // Starts a copy from the backing store or the composited surface. Must be run 240 // on the UI BrowserThread. |deliver_frame_cb| will be run when the operation 241 // completes. The copy will occur to |target|. 242 // 243 // This may be used as a ContentCaptureSubscription::CaptureCallback. 244 void Capture(const base::TimeTicks& start_time, 245 const scoped_refptr<media::VideoFrame>& target, 246 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& 247 deliver_frame_cb); 248 249 private: 250 bool IsStarted() const; 251 252 // Computes the preferred size of the target RenderWidget for optimal capture. 253 gfx::Size ComputeOptimalTargetSize() const; 254 255 // Response callback for RenderWidgetHost::CopyFromBackingStore(). 256 void DidCopyFromBackingStore( 257 const base::TimeTicks& start_time, 258 const scoped_refptr<media::VideoFrame>& target, 259 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& 260 deliver_frame_cb, 261 bool success, 262 const SkBitmap& bitmap); 263 264 // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame(). 265 void DidCopyFromCompositingSurfaceToVideoFrame( 266 const base::TimeTicks& start_time, 267 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& 268 deliver_frame_cb, 269 bool success); 270 271 // Remove the old subscription, and start a new one if |rwh| is not NULL. 272 void RenewFrameSubscription(RenderWidgetHost* rwh); 273 274 // Parameters saved in constructor. 275 const int initial_render_process_id_; 276 const int initial_main_render_frame_id_; 277 278 // Tracks events and calls back to RenewFrameSubscription() to maintain 279 // capture on the correct RenderWidgetHost. 280 const scoped_refptr<WebContentsTracker> tracker_; 281 282 // A dedicated worker thread on which SkBitmap->VideoFrame conversion will 283 // occur. Only used when this activity cannot be done on the GPU. 284 scoped_ptr<base::Thread> render_thread_; 285 286 // Makes all the decisions about which frames to copy, and how. 287 scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_; 288 289 // Video capture parameters that this machine is started with. 290 media::VideoCaptureParams capture_params_; 291 292 // Last known RenderView size. 293 gfx::Size last_view_size_; 294 295 // Responsible for forwarding events from the active RenderWidgetHost to the 296 // oracle, and initiating captures accordingly. 297 scoped_ptr<ContentCaptureSubscription> subscription_; 298 299 // Weak pointer factory used to invalidate callbacks. 300 // NOTE: Weak pointers must be invalidated before all other member variables. 301 base::WeakPtrFactory<WebContentsCaptureMachine> weak_ptr_factory_; 302 303 DISALLOW_COPY_AND_ASSIGN(WebContentsCaptureMachine); 304 }; 305 306 bool FrameSubscriber::ShouldCaptureFrame( 307 const gfx::Rect& damage_rect, 308 base::TimeTicks present_time, 309 scoped_refptr<media::VideoFrame>* storage, 310 DeliverFrameCallback* deliver_frame_cb) { 311 TRACE_EVENT1("mirroring", "FrameSubscriber::ShouldCaptureFrame", 312 "instance", this); 313 314 ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb; 315 bool oracle_decision = oracle_proxy_->ObserveEventAndDecideCapture( 316 event_type_, damage_rect, present_time, storage, &capture_frame_cb); 317 318 if (!capture_frame_cb.is_null()) 319 *deliver_frame_cb = base::Bind(capture_frame_cb, *storage); 320 if (oracle_decision) 321 delivery_log_->ChronicleFrameDelivery(present_time); 322 return oracle_decision; 323 } 324 325 ContentCaptureSubscription::ContentCaptureSubscription( 326 const RenderWidgetHost& source, 327 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy, 328 const CaptureCallback& capture_callback) 329 : render_process_id_(source.GetProcess()->GetID()), 330 render_widget_id_(source.GetRoutingID()), 331 delivery_log_(), 332 paint_subscriber_(VideoCaptureOracle::kSoftwarePaint, oracle_proxy, 333 &delivery_log_), 334 timer_subscriber_(VideoCaptureOracle::kTimerPoll, oracle_proxy, 335 &delivery_log_), 336 capture_callback_(capture_callback), 337 timer_(true, true) { 338 DCHECK_CURRENTLY_ON(BrowserThread::UI); 339 340 RenderWidgetHostView* const view = source.GetView(); 341 342 // Subscribe to accelerated presents. These will be serviced directly by the 343 // oracle. 344 if (view && kAcceleratedSubscriberIsSupported) { 345 scoped_ptr<RenderWidgetHostViewFrameSubscriber> subscriber( 346 new FrameSubscriber(VideoCaptureOracle::kCompositorUpdate, 347 oracle_proxy, &delivery_log_)); 348 view->BeginFrameSubscription(subscriber.Pass()); 349 } 350 351 // Subscribe to software paint events. This instance will service these by 352 // reflecting them back to the WebContentsCaptureMachine via 353 // |capture_callback|. 354 registrar_.Add( 355 this, content::NOTIFICATION_RENDER_WIDGET_HOST_DID_UPDATE_BACKING_STORE, 356 Source<RenderWidgetHost>(&source)); 357 358 // Subscribe to timer events. This instance will service these as well. 359 timer_.Start(FROM_HERE, oracle_proxy->min_capture_period(), 360 base::Bind(&ContentCaptureSubscription::OnTimer, 361 base::Unretained(this))); 362 } 363 364 ContentCaptureSubscription::~ContentCaptureSubscription() { 365 // If the BrowserThreads have been torn down, then the browser is in the final 366 // stages of exiting and it is dangerous to take any further action. We must 367 // return early. http://crbug.com/396413 368 if (!BrowserThread::IsMessageLoopValid(BrowserThread::UI)) 369 return; 370 371 DCHECK_CURRENTLY_ON(BrowserThread::UI); 372 if (kAcceleratedSubscriberIsSupported) { 373 RenderWidgetHost* const source = 374 RenderWidgetHost::FromID(render_process_id_, render_widget_id_); 375 RenderWidgetHostView* const view = source ? source->GetView() : NULL; 376 if (view) 377 view->EndFrameSubscription(); 378 } 379 } 380 381 void ContentCaptureSubscription::Observe( 382 int type, 383 const content::NotificationSource& source, 384 const content::NotificationDetails& details) { 385 DCHECK_CURRENTLY_ON(BrowserThread::UI); 386 DCHECK_EQ(NOTIFICATION_RENDER_WIDGET_HOST_DID_UPDATE_BACKING_STORE, type); 387 388 RenderWidgetHostImpl* rwh = 389 RenderWidgetHostImpl::From(Source<RenderWidgetHost>(source).ptr()); 390 391 // This message occurs on window resizes and visibility changes even when 392 // accelerated compositing is active, so we need to filter out these cases. 393 if (!rwh || !rwh->GetView()) 394 return; 395 // Mac sends DID_UPDATE_BACKING_STORE messages to inform the capture system 396 // of new software compositor frames, so always treat these messages as 397 // signals of a new frame on Mac. 398 // http://crbug.com/333986 399 #if !defined(OS_MACOSX) 400 if (rwh->GetView()->IsSurfaceAvailableForCopy()) 401 return; 402 #endif 403 404 TRACE_EVENT1("mirroring", "ContentCaptureSubscription::Observe", 405 "instance", this); 406 407 base::Closure copy_done_callback; 408 scoped_refptr<media::VideoFrame> frame; 409 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb; 410 const base::TimeTicks start_time = base::TimeTicks::Now(); 411 if (paint_subscriber_.ShouldCaptureFrame(gfx::Rect(), 412 start_time, 413 &frame, 414 &deliver_frame_cb)) { 415 // This message happens just before paint. If we post a task to do the copy, 416 // it should run soon after the paint. 417 BrowserThread::PostTask( 418 BrowserThread::UI, FROM_HERE, 419 base::Bind(capture_callback_, start_time, frame, deliver_frame_cb)); 420 } 421 } 422 423 void ContentCaptureSubscription::OnTimer() { 424 DCHECK_CURRENTLY_ON(BrowserThread::UI); 425 TRACE_EVENT0("mirroring", "ContentCaptureSubscription::OnTimer"); 426 427 scoped_refptr<media::VideoFrame> frame; 428 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb; 429 430 const base::TimeTicks start_time = base::TimeTicks::Now(); 431 if (timer_subscriber_.ShouldCaptureFrame(gfx::Rect(), 432 start_time, 433 &frame, 434 &deliver_frame_cb)) { 435 capture_callback_.Run(start_time, frame, deliver_frame_cb); 436 } 437 } 438 439 void RenderVideoFrame(const SkBitmap& input, 440 const scoped_refptr<media::VideoFrame>& output, 441 const base::Callback<void(bool)>& done_cb) { 442 base::ScopedClosureRunner failure_handler(base::Bind(done_cb, false)); 443 444 SkAutoLockPixels locker(input); 445 446 // Sanity-check the captured bitmap. 447 if (input.empty() || 448 !input.readyToDraw() || 449 input.colorType() != kN32_SkColorType || 450 input.width() < 2 || input.height() < 2) { 451 DVLOG(1) << "input unacceptable (size=" 452 << input.getSize() 453 << ", ready=" << input.readyToDraw() 454 << ", colorType=" << input.colorType() << ')'; 455 return; 456 } 457 458 // Sanity-check the output buffer. 459 if (output->format() != media::VideoFrame::I420) { 460 NOTREACHED(); 461 return; 462 } 463 464 // Calculate the width and height of the content region in the |output|, based 465 // on the aspect ratio of |input|. 466 gfx::Rect region_in_frame = ComputeYV12LetterboxRegion( 467 output->coded_size(), gfx::Size(input.width(), input.height())); 468 469 // Scale the bitmap to the required size, if necessary. 470 SkBitmap scaled_bitmap; 471 if (input.width() != region_in_frame.width() || 472 input.height() != region_in_frame.height()) { 473 474 skia::ImageOperations::ResizeMethod method; 475 if (input.width() < region_in_frame.width() || 476 input.height() < region_in_frame.height()) { 477 // Avoid box filtering when magnifying, because it's actually 478 // nearest-neighbor. 479 method = skia::ImageOperations::RESIZE_HAMMING1; 480 } else { 481 method = skia::ImageOperations::RESIZE_BOX; 482 } 483 484 TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", output.get(), "Scale"); 485 scaled_bitmap = skia::ImageOperations::Resize(input, method, 486 region_in_frame.width(), 487 region_in_frame.height()); 488 } else { 489 scaled_bitmap = input; 490 } 491 492 TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", output.get(), "YUV"); 493 { 494 SkAutoLockPixels scaled_bitmap_locker(scaled_bitmap); 495 496 media::CopyRGBToVideoFrame( 497 reinterpret_cast<uint8*>(scaled_bitmap.getPixels()), 498 scaled_bitmap.rowBytes(), 499 region_in_frame, 500 output.get()); 501 } 502 503 // The result is now ready. 504 ignore_result(failure_handler.Release()); 505 done_cb.Run(true); 506 } 507 508 VideoFrameDeliveryLog::VideoFrameDeliveryLog() 509 : last_frame_rate_log_time_(), 510 count_frames_rendered_(0) { 511 } 512 513 void VideoFrameDeliveryLog::ChronicleFrameDelivery(base::TimeTicks frame_time) { 514 // Log frame rate, if verbose logging is turned on. 515 static const base::TimeDelta kFrameRateLogInterval = 516 base::TimeDelta::FromSeconds(10); 517 if (last_frame_rate_log_time_.is_null()) { 518 last_frame_rate_log_time_ = frame_time; 519 count_frames_rendered_ = 0; 520 } else { 521 ++count_frames_rendered_; 522 const base::TimeDelta elapsed = frame_time - last_frame_rate_log_time_; 523 if (elapsed >= kFrameRateLogInterval) { 524 const double measured_fps = 525 count_frames_rendered_ / elapsed.InSecondsF(); 526 UMA_HISTOGRAM_COUNTS( 527 "TabCapture.FrameRate", 528 static_cast<int>(measured_fps)); 529 VLOG(1) << "Current measured frame rate for " 530 << "WebContentsVideoCaptureDevice is " << measured_fps << " FPS."; 531 last_frame_rate_log_time_ = frame_time; 532 count_frames_rendered_ = 0; 533 } 534 } 535 } 536 537 WebContentsCaptureMachine::WebContentsCaptureMachine(int render_process_id, 538 int main_render_frame_id) 539 : initial_render_process_id_(render_process_id), 540 initial_main_render_frame_id_(main_render_frame_id), 541 tracker_(new WebContentsTracker(true)), 542 weak_ptr_factory_(this) {} 543 544 WebContentsCaptureMachine::~WebContentsCaptureMachine() {} 545 546 bool WebContentsCaptureMachine::IsStarted() const { 547 DCHECK_CURRENTLY_ON(BrowserThread::UI); 548 return weak_ptr_factory_.HasWeakPtrs(); 549 } 550 551 bool WebContentsCaptureMachine::Start( 552 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy, 553 const media::VideoCaptureParams& params) { 554 DCHECK_CURRENTLY_ON(BrowserThread::UI); 555 DCHECK(!IsStarted()); 556 557 DCHECK(oracle_proxy.get()); 558 oracle_proxy_ = oracle_proxy; 559 capture_params_ = params; 560 561 render_thread_.reset(new base::Thread("WebContentsVideo_RenderThread")); 562 if (!render_thread_->Start()) { 563 DVLOG(1) << "Failed to spawn render thread."; 564 render_thread_.reset(); 565 return false; 566 } 567 568 // Note: Creation of the first WeakPtr in the following statement will cause 569 // IsStarted() to return true from now on. 570 tracker_->Start(initial_render_process_id_, initial_main_render_frame_id_, 571 base::Bind(&WebContentsCaptureMachine::RenewFrameSubscription, 572 weak_ptr_factory_.GetWeakPtr())); 573 574 return true; 575 } 576 577 void WebContentsCaptureMachine::Stop(const base::Closure& callback) { 578 DCHECK_CURRENTLY_ON(BrowserThread::UI); 579 580 if (!IsStarted()) { 581 callback.Run(); 582 return; 583 } 584 585 // The following cancels any outstanding callbacks and causes IsStarted() to 586 // return false from here onward. 587 weak_ptr_factory_.InvalidateWeakPtrs(); 588 589 // Note: RenewFrameSubscription() must be called before stopping |tracker_| so 590 // the web_contents() can be notified that the capturing is ending. 591 RenewFrameSubscription(NULL); 592 tracker_->Stop(); 593 594 // The render thread cannot be stopped on the UI thread, so post a message 595 // to the thread pool used for blocking operations. 596 if (render_thread_.get()) { 597 BrowserThread::PostBlockingPoolTask( 598 FROM_HERE, 599 base::Bind(&DeleteOnWorkerThread, base::Passed(&render_thread_), 600 callback)); 601 } 602 } 603 604 void WebContentsCaptureMachine::Capture( 605 const base::TimeTicks& start_time, 606 const scoped_refptr<media::VideoFrame>& target, 607 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& 608 deliver_frame_cb) { 609 DCHECK_CURRENTLY_ON(BrowserThread::UI); 610 611 RenderWidgetHost* rwh = tracker_->GetTargetRenderWidgetHost(); 612 RenderWidgetHostViewBase* view = 613 rwh ? static_cast<RenderWidgetHostViewBase*>(rwh->GetView()) : NULL; 614 if (!view) { 615 deliver_frame_cb.Run(base::TimeTicks(), false); 616 return; 617 } 618 619 gfx::Size video_size = target->coded_size(); 620 gfx::Size view_size = view->GetViewBounds().size(); 621 gfx::Size fitted_size; 622 if (!view_size.IsEmpty()) { 623 fitted_size = ComputeYV12LetterboxRegion(video_size, view_size).size(); 624 } 625 if (view_size != last_view_size_) { 626 last_view_size_ = view_size; 627 628 // Measure the number of kilopixels. 629 UMA_HISTOGRAM_COUNTS_10000( 630 "TabCapture.ViewChangeKiloPixels", 631 view_size.width() * view_size.height() / 1024); 632 } 633 634 if (view->CanCopyToVideoFrame()) { 635 view->CopyFromCompositingSurfaceToVideoFrame( 636 gfx::Rect(view_size), 637 target, 638 base::Bind(&WebContentsCaptureMachine:: 639 DidCopyFromCompositingSurfaceToVideoFrame, 640 weak_ptr_factory_.GetWeakPtr(), 641 start_time, deliver_frame_cb)); 642 } else { 643 rwh->CopyFromBackingStore( 644 gfx::Rect(), 645 fitted_size, // Size here is a request not always honored. 646 base::Bind(&WebContentsCaptureMachine::DidCopyFromBackingStore, 647 weak_ptr_factory_.GetWeakPtr(), 648 start_time, 649 target, 650 deliver_frame_cb), 651 kN32_SkColorType); 652 } 653 } 654 655 gfx::Size WebContentsCaptureMachine::ComputeOptimalTargetSize() const { 656 DCHECK_CURRENTLY_ON(BrowserThread::UI); 657 658 gfx::Size optimal_size = oracle_proxy_->GetCaptureSize(); 659 660 // If the ratio between physical and logical pixels is greater than 1:1, 661 // shrink |optimal_size| by that amount. Then, when external code resizes the 662 // render widget to the "preferred size," the widget will be physically 663 // rendered at the exact capture size, thereby eliminating unnecessary scaling 664 // operations in the graphics pipeline. 665 RenderWidgetHost* const rwh = tracker_->GetTargetRenderWidgetHost(); 666 RenderWidgetHostView* const rwhv = rwh ? rwh->GetView() : NULL; 667 if (rwhv) { 668 const gfx::NativeView view = rwhv->GetNativeView(); 669 gfx::Screen* const screen = gfx::Screen::GetScreenFor(view); 670 if (screen->IsDIPEnabled()) { 671 const gfx::Display display = screen->GetDisplayNearestWindow(view); 672 const float scale = display.device_scale_factor(); 673 if (scale > 1.0f) { 674 const gfx::Size shrunk_size( 675 gfx::ToFlooredSize(gfx::ScaleSize(optimal_size, 1.0f / scale))); 676 if (shrunk_size.width() > 0 && shrunk_size.height() > 0) 677 optimal_size = shrunk_size; 678 } 679 } 680 } 681 682 VLOG(1) << "Computed optimal target size: " << optimal_size.ToString(); 683 return optimal_size; 684 } 685 686 void WebContentsCaptureMachine::DidCopyFromBackingStore( 687 const base::TimeTicks& start_time, 688 const scoped_refptr<media::VideoFrame>& target, 689 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& 690 deliver_frame_cb, 691 bool success, 692 const SkBitmap& bitmap) { 693 DCHECK_CURRENTLY_ON(BrowserThread::UI); 694 695 base::TimeTicks now = base::TimeTicks::Now(); 696 DCHECK(render_thread_.get()); 697 if (success) { 698 UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeBitmap", now - start_time); 699 TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", target.get(), 700 "Render"); 701 render_thread_->message_loop_proxy()->PostTask(FROM_HERE, base::Bind( 702 &RenderVideoFrame, bitmap, target, 703 base::Bind(deliver_frame_cb, start_time))); 704 } else { 705 // Capture can fail due to transient issues, so just skip this frame. 706 DVLOG(1) << "CopyFromBackingStore failed; skipping frame."; 707 deliver_frame_cb.Run(start_time, false); 708 } 709 } 710 711 void WebContentsCaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame( 712 const base::TimeTicks& start_time, 713 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& 714 deliver_frame_cb, 715 bool success) { 716 DCHECK_CURRENTLY_ON(BrowserThread::UI); 717 base::TimeTicks now = base::TimeTicks::Now(); 718 719 if (success) { 720 UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeVideoFrame", now - start_time); 721 } else { 722 // Capture can fail due to transient issues, so just skip this frame. 723 DVLOG(1) << "CopyFromCompositingSurface failed; skipping frame."; 724 } 725 deliver_frame_cb.Run(start_time, success); 726 } 727 728 void WebContentsCaptureMachine::RenewFrameSubscription(RenderWidgetHost* rwh) { 729 DCHECK_CURRENTLY_ON(BrowserThread::UI); 730 731 // Always destroy the old subscription before creating a new one. 732 const bool had_subscription = !!subscription_; 733 subscription_.reset(); 734 735 DVLOG(1) << "Renewing frame subscription to RWH@" << rwh 736 << ", had_subscription=" << had_subscription; 737 738 if (!rwh) { 739 if (had_subscription && tracker_->web_contents()) 740 tracker_->web_contents()->DecrementCapturerCount(); 741 if (IsStarted()) { 742 // Tracking of WebContents and/or its main frame has failed before Stop() 743 // was called, so report this as an error: 744 oracle_proxy_->ReportError("WebContents and/or main frame are gone."); 745 } 746 return; 747 } 748 749 if (!had_subscription && tracker_->web_contents()) { 750 tracker_->web_contents()->IncrementCapturerCount( 751 ComputeOptimalTargetSize()); 752 } 753 754 subscription_.reset(new ContentCaptureSubscription(*rwh, oracle_proxy_, 755 base::Bind(&WebContentsCaptureMachine::Capture, 756 weak_ptr_factory_.GetWeakPtr()))); 757 } 758 759 } // namespace 760 761 WebContentsVideoCaptureDevice::WebContentsVideoCaptureDevice( 762 int render_process_id, int main_render_frame_id) 763 : core_(new ContentVideoCaptureDeviceCore(scoped_ptr<VideoCaptureMachine>( 764 new WebContentsCaptureMachine( 765 render_process_id, main_render_frame_id)))) {} 766 767 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() { 768 DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying."; 769 } 770 771 // static 772 media::VideoCaptureDevice* WebContentsVideoCaptureDevice::Create( 773 const std::string& device_id) { 774 // Parse device_id into render_process_id and main_render_frame_id. 775 int render_process_id = -1; 776 int main_render_frame_id = -1; 777 if (!WebContentsCaptureUtil::ExtractTabCaptureTarget( 778 device_id, &render_process_id, &main_render_frame_id)) { 779 return NULL; 780 } 781 782 return new WebContentsVideoCaptureDevice( 783 render_process_id, main_render_frame_id); 784 } 785 786 void WebContentsVideoCaptureDevice::AllocateAndStart( 787 const media::VideoCaptureParams& params, 788 scoped_ptr<Client> client) { 789 DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString(); 790 core_->AllocateAndStart(params, client.Pass()); 791 } 792 793 void WebContentsVideoCaptureDevice::StopAndDeAllocate() { 794 core_->StopAndDeAllocate(); 795 } 796 797 } // namespace content 798