1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 // 5 // Implementation notes: This needs to work on a variety of hardware 6 // configurations where the speed of the CPU and GPU greatly affect overall 7 // performance. Spanning several threads, the process of capturing has been 8 // split up into four conceptual stages: 9 // 10 // 1. Reserve Buffer: Before a frame can be captured, a slot in the client's 11 // shared-memory IPC buffer is reserved. There are only a few of these; 12 // when they run out, it indicates that the downstream client -- likely a 13 // video encoder -- is the performance bottleneck, and that the rate of 14 // frame capture should be throttled back. 15 // 16 // 2. Capture: A bitmap is snapshotted/copied from the RenderView's backing 17 // store. This is initiated on the UI BrowserThread, and often occurs 18 // asynchronously. Where supported, the GPU scales and color converts 19 // frames to our desired size, and the readback happens directly into the 20 // shared-memory buffer. But this is not always possible, particularly when 21 // accelerated compositing is disabled. 22 // 23 // 3. Render (if needed): If the web contents cannot be captured directly into 24 // our target size and color format, scaling and colorspace conversion must 25 // be done on the CPU. A dedicated thread is used for this operation, to 26 // avoid blocking the UI thread. The Render stage always reads from a 27 // bitmap returned by Capture, and writes into the reserved slot in the 28 // shared-memory buffer. 29 // 30 // 4. Deliver: The rendered video frame is returned to the client (which 31 // implements the VideoCaptureDevice::Client interface). Because all 32 // paths have written the frame into the IPC buffer, this step should 33 // never need to do an additional copy of the pixel data. 34 // 35 // In the best-performing case, the Render step is bypassed: Capture produces 36 // ready-to-Deliver frames. But when accelerated readback is not possible, the 37 // system is designed so that Capture and Render may run concurrently. A timing 38 // diagram helps illustrate this point (@30 FPS): 39 // 40 // Time: 0ms 33ms 66ms 99ms 41 // thread1: |-Capture-f1------v |-Capture-f2------v |-Capture-f3----v |-Capt 42 // thread2: |-Render-f1-----v |-Render-f2-----v |-Render-f3 43 // 44 // In the above example, both capturing and rendering *each* take almost the 45 // full 33 ms available between frames, yet we see that the required throughput 46 // is obtained. 47 // 48 // Turning on verbose logging will cause the effective frame rate to be logged 49 // at 5-second intervals. 50 51 #include "content/browser/media/capture/web_contents_video_capture_device.h" 52 53 #include "base/basictypes.h" 54 #include "base/bind.h" 55 #include "base/callback_helpers.h" 56 #include "base/logging.h" 57 #include "base/memory/scoped_ptr.h" 58 #include "base/memory/weak_ptr.h" 59 #include "base/message_loop/message_loop_proxy.h" 60 #include "base/metrics/histogram.h" 61 #include "base/sequenced_task_runner.h" 62 #include "base/threading/thread.h" 63 #include "base/threading/thread_checker.h" 64 #include "base/time/time.h" 65 #include "content/browser/media/capture/content_video_capture_device_core.h" 66 #include "content/browser/media/capture/video_capture_oracle.h" 67 #include "content/browser/media/capture/web_contents_capture_util.h" 68 #include "content/browser/renderer_host/render_widget_host_impl.h" 69 #include "content/browser/renderer_host/render_widget_host_view_base.h" 70 #include "content/browser/web_contents/web_contents_impl.h" 71 #include "content/public/browser/browser_thread.h" 72 #include "content/public/browser/notification_source.h" 73 #include "content/public/browser/notification_types.h" 74 #include "content/public/browser/render_view_host.h" 75 #include "content/public/browser/render_widget_host_view.h" 76 #include "content/public/browser/render_widget_host_view_frame_subscriber.h" 77 #include "content/public/browser/web_contents_observer.h" 78 #include "media/base/video_util.h" 79 #include "media/video/capture/video_capture_types.h" 80 #include "skia/ext/image_operations.h" 81 #include "third_party/skia/include/core/SkBitmap.h" 82 #include "third_party/skia/include/core/SkColor.h" 83 84 namespace content { 85 86 namespace { 87 88 // Compute a letterbox region, aligned to even coordinates. 89 gfx::Rect ComputeYV12LetterboxRegion(const gfx::Size& frame_size, 90 const gfx::Size& content_size) { 91 92 gfx::Rect result = media::ComputeLetterboxRegion(gfx::Rect(frame_size), 93 content_size); 94 95 result.set_x(MakeEven(result.x())); 96 result.set_y(MakeEven(result.y())); 97 result.set_width(std::max(kMinFrameWidth, MakeEven(result.width()))); 98 result.set_height(std::max(kMinFrameHeight, MakeEven(result.height()))); 99 100 return result; 101 } 102 103 void DeleteOnWorkerThread(scoped_ptr<base::Thread> render_thread, 104 const base::Closure& callback) { 105 render_thread.reset(); 106 107 // After thread join call the callback on UI thread. 108 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, callback); 109 } 110 111 // Responsible for logging the effective frame rate. 112 class VideoFrameDeliveryLog { 113 public: 114 VideoFrameDeliveryLog(); 115 116 // Report that the frame posted with |frame_time| has been delivered. 117 void ChronicleFrameDelivery(base::TimeTicks frame_time); 118 119 private: 120 // The following keep track of and log the effective frame rate whenever 121 // verbose logging is turned on. 122 base::TimeTicks last_frame_rate_log_time_; 123 int count_frames_rendered_; 124 125 DISALLOW_COPY_AND_ASSIGN(VideoFrameDeliveryLog); 126 }; 127 128 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible 129 // with RenderWidgetHostViewFrameSubscriber. We create one per event type. 130 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber { 131 public: 132 FrameSubscriber(VideoCaptureOracle::Event event_type, 133 const scoped_refptr<ThreadSafeCaptureOracle>& oracle, 134 VideoFrameDeliveryLog* delivery_log) 135 : event_type_(event_type), 136 oracle_proxy_(oracle), 137 delivery_log_(delivery_log) {} 138 139 virtual bool ShouldCaptureFrame( 140 base::TimeTicks present_time, 141 scoped_refptr<media::VideoFrame>* storage, 142 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback* 143 deliver_frame_cb) OVERRIDE; 144 145 private: 146 const VideoCaptureOracle::Event event_type_; 147 scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_; 148 VideoFrameDeliveryLog* const delivery_log_; 149 }; 150 151 // ContentCaptureSubscription is the relationship between a RenderWidgetHost 152 // whose content is updating, a subscriber that is deciding which of these 153 // updates to capture (and where to deliver them to), and a callback that 154 // knows how to do the capture and prepare the result for delivery. 155 // 156 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in 157 // the RenderWidgetHostView, to process updates that occur via accelerated 158 // compositing, (b) installing itself as an observer of updates to the 159 // RenderWidgetHost's backing store, to hook updates that occur via software 160 // rendering, and (c) running a timer to possibly initiate non-event-driven 161 // captures that the subscriber might request. 162 // 163 // All of this happens on the UI thread, although the 164 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates 165 // autonomously on some other thread. 166 class ContentCaptureSubscription : public content::NotificationObserver { 167 public: 168 typedef base::Callback< 169 void(const base::TimeTicks&, 170 const scoped_refptr<media::VideoFrame>&, 171 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&)> 172 CaptureCallback; 173 174 // Create a subscription. Whenever a manual capture is required, the 175 // subscription will invoke |capture_callback| on the UI thread to do the 176 // work. 177 ContentCaptureSubscription( 178 const RenderWidgetHost& source, 179 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy, 180 const CaptureCallback& capture_callback); 181 virtual ~ContentCaptureSubscription(); 182 183 // content::NotificationObserver implementation. 184 virtual void Observe(int type, 185 const content::NotificationSource& source, 186 const content::NotificationDetails& details) OVERRIDE; 187 188 private: 189 void OnTimer(); 190 191 const int render_process_id_; 192 const int render_view_id_; 193 194 VideoFrameDeliveryLog delivery_log_; 195 FrameSubscriber paint_subscriber_; 196 FrameSubscriber timer_subscriber_; 197 content::NotificationRegistrar registrar_; 198 CaptureCallback capture_callback_; 199 base::Timer timer_; 200 201 DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription); 202 }; 203 204 // Render the SkBitmap |input| into the given VideoFrame buffer |output|, then 205 // invoke |done_cb| to indicate success or failure. |input| is expected to be 206 // ARGB. |output| must be YV12 or I420. Colorspace conversion is always done. 207 // Scaling and letterboxing will be done as needed. 208 // 209 // This software implementation should be used only when GPU acceleration of 210 // these activities is not possible. This operation may be expensive (tens to 211 // hundreds of milliseconds), so the caller should ensure that it runs on a 212 // thread where such a pause would cause UI jank. 213 void RenderVideoFrame(const SkBitmap& input, 214 const scoped_refptr<media::VideoFrame>& output, 215 const base::Callback<void(bool)>& done_cb); 216 217 // Keeps track of the RenderView to be sourced, and executes copying of the 218 // backing store on the UI BrowserThread. 219 // 220 // TODO(nick): It would be nice to merge this with WebContentsTracker, but its 221 // implementation is currently asynchronous -- in our case, the "rvh changed" 222 // notification would get posted back to the UI thread and processed later, and 223 // this seems disadvantageous. 224 class WebContentsCaptureMachine 225 : public VideoCaptureMachine, 226 public WebContentsObserver { 227 public: 228 WebContentsCaptureMachine(int render_process_id, int render_view_id); 229 virtual ~WebContentsCaptureMachine(); 230 231 // VideoCaptureMachine overrides. 232 virtual bool Start(const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy, 233 const media::VideoCaptureParams& params) OVERRIDE; 234 virtual void Stop(const base::Closure& callback) OVERRIDE; 235 236 // Starts a copy from the backing store or the composited surface. Must be run 237 // on the UI BrowserThread. |deliver_frame_cb| will be run when the operation 238 // completes. The copy will occur to |target|. 239 // 240 // This may be used as a ContentCaptureSubscription::CaptureCallback. 241 void Capture(const base::TimeTicks& start_time, 242 const scoped_refptr<media::VideoFrame>& target, 243 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& 244 deliver_frame_cb); 245 246 // content::WebContentsObserver implementation. 247 virtual void DidShowFullscreenWidget(int routing_id) OVERRIDE { 248 fullscreen_widget_id_ = routing_id; 249 RenewFrameSubscription(); 250 } 251 252 virtual void DidDestroyFullscreenWidget(int routing_id) OVERRIDE { 253 DCHECK_EQ(fullscreen_widget_id_, routing_id); 254 fullscreen_widget_id_ = MSG_ROUTING_NONE; 255 RenewFrameSubscription(); 256 } 257 258 virtual void RenderViewReady() OVERRIDE { 259 RenewFrameSubscription(); 260 } 261 262 virtual void AboutToNavigateRenderView(RenderViewHost* rvh) OVERRIDE { 263 RenewFrameSubscription(); 264 } 265 266 virtual void DidNavigateMainFrame( 267 const LoadCommittedDetails& details, 268 const FrameNavigateParams& params) OVERRIDE { 269 RenewFrameSubscription(); 270 } 271 272 virtual void WebContentsDestroyed() OVERRIDE; 273 274 private: 275 // Starts observing the web contents, returning false if lookup fails. 276 bool StartObservingWebContents(); 277 278 // Helper function to determine the view that we are currently tracking. 279 RenderWidgetHost* GetTarget(); 280 281 // Response callback for RenderWidgetHost::CopyFromBackingStore(). 282 void DidCopyFromBackingStore( 283 const base::TimeTicks& start_time, 284 const scoped_refptr<media::VideoFrame>& target, 285 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& 286 deliver_frame_cb, 287 bool success, 288 const SkBitmap& bitmap); 289 290 // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame(). 291 void DidCopyFromCompositingSurfaceToVideoFrame( 292 const base::TimeTicks& start_time, 293 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& 294 deliver_frame_cb, 295 bool success); 296 297 // Remove the old subscription, and start a new one. This should be called 298 // after any change to the WebContents that affects the RenderWidgetHost or 299 // attached views. 300 void RenewFrameSubscription(); 301 302 // Parameters saved in constructor. 303 const int initial_render_process_id_; 304 const int initial_render_view_id_; 305 306 // A dedicated worker thread on which SkBitmap->VideoFrame conversion will 307 // occur. Only used when this activity cannot be done on the GPU. 308 scoped_ptr<base::Thread> render_thread_; 309 310 // Makes all the decisions about which frames to copy, and how. 311 scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_; 312 313 // Video capture parameters that this machine is started with. 314 media::VideoCaptureParams capture_params_; 315 316 // Routing ID of any active fullscreen render widget or MSG_ROUTING_NONE 317 // otherwise. 318 int fullscreen_widget_id_; 319 320 // Last known RenderView size. 321 gfx::Size last_view_size_; 322 323 // Responsible for forwarding events from the active RenderWidgetHost to the 324 // oracle, and initiating captures accordingly. 325 scoped_ptr<ContentCaptureSubscription> subscription_; 326 327 // Weak pointer factory used to invalidate callbacks. 328 // NOTE: Weak pointers must be invalidated before all other member variables. 329 base::WeakPtrFactory<WebContentsCaptureMachine> weak_ptr_factory_; 330 331 DISALLOW_COPY_AND_ASSIGN(WebContentsCaptureMachine); 332 }; 333 334 bool FrameSubscriber::ShouldCaptureFrame( 335 base::TimeTicks present_time, 336 scoped_refptr<media::VideoFrame>* storage, 337 DeliverFrameCallback* deliver_frame_cb) { 338 TRACE_EVENT1("mirroring", "FrameSubscriber::ShouldCaptureFrame", 339 "instance", this); 340 341 ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb; 342 bool oracle_decision = oracle_proxy_->ObserveEventAndDecideCapture( 343 event_type_, present_time, storage, &capture_frame_cb); 344 345 if (!capture_frame_cb.is_null()) 346 *deliver_frame_cb = base::Bind(capture_frame_cb, *storage); 347 if (oracle_decision) 348 delivery_log_->ChronicleFrameDelivery(present_time); 349 return oracle_decision; 350 } 351 352 ContentCaptureSubscription::ContentCaptureSubscription( 353 const RenderWidgetHost& source, 354 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy, 355 const CaptureCallback& capture_callback) 356 : render_process_id_(source.GetProcess()->GetID()), 357 render_view_id_(source.GetRoutingID()), 358 delivery_log_(), 359 paint_subscriber_(VideoCaptureOracle::kSoftwarePaint, oracle_proxy, 360 &delivery_log_), 361 timer_subscriber_(VideoCaptureOracle::kTimerPoll, oracle_proxy, 362 &delivery_log_), 363 capture_callback_(capture_callback), 364 timer_(true, true) { 365 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 366 367 RenderWidgetHostViewBase* view = static_cast<RenderWidgetHostViewBase*>( 368 source.GetView()); 369 370 // Subscribe to accelerated presents. These will be serviced directly by the 371 // oracle. 372 if (view && kAcceleratedSubscriberIsSupported) { 373 scoped_ptr<RenderWidgetHostViewFrameSubscriber> subscriber( 374 new FrameSubscriber(VideoCaptureOracle::kCompositorUpdate, 375 oracle_proxy, &delivery_log_)); 376 view->BeginFrameSubscription(subscriber.Pass()); 377 } 378 379 // Subscribe to software paint events. This instance will service these by 380 // reflecting them back to the WebContentsCaptureMachine via 381 // |capture_callback|. 382 registrar_.Add( 383 this, content::NOTIFICATION_RENDER_WIDGET_HOST_DID_UPDATE_BACKING_STORE, 384 Source<RenderWidgetHost>(&source)); 385 386 // Subscribe to timer events. This instance will service these as well. 387 timer_.Start(FROM_HERE, oracle_proxy->capture_period(), 388 base::Bind(&ContentCaptureSubscription::OnTimer, 389 base::Unretained(this))); 390 } 391 392 ContentCaptureSubscription::~ContentCaptureSubscription() { 393 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 394 if (kAcceleratedSubscriberIsSupported) { 395 RenderViewHost* source = RenderViewHost::FromID(render_process_id_, 396 render_view_id_); 397 if (source) { 398 RenderWidgetHostViewBase* view = static_cast<RenderWidgetHostViewBase*>( 399 source->GetView()); 400 if (view) 401 view->EndFrameSubscription(); 402 } 403 } 404 } 405 406 void ContentCaptureSubscription::Observe( 407 int type, 408 const content::NotificationSource& source, 409 const content::NotificationDetails& details) { 410 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 411 DCHECK_EQ(NOTIFICATION_RENDER_WIDGET_HOST_DID_UPDATE_BACKING_STORE, type); 412 413 RenderWidgetHostImpl* rwh = 414 RenderWidgetHostImpl::From(Source<RenderWidgetHost>(source).ptr()); 415 416 // This message occurs on window resizes and visibility changes even when 417 // accelerated compositing is active, so we need to filter out these cases. 418 if (!rwh || !rwh->GetView()) 419 return; 420 // Mac sends DID_UPDATE_BACKING_STORE messages to inform the capture system 421 // of new software compositor frames, so always treat these messages as 422 // signals of a new frame on Mac. 423 // http://crbug.com/333986 424 #if !defined(OS_MACOSX) 425 if (rwh->GetView()->IsSurfaceAvailableForCopy()) 426 return; 427 #endif 428 429 TRACE_EVENT1("mirroring", "ContentCaptureSubscription::Observe", 430 "instance", this); 431 432 base::Closure copy_done_callback; 433 scoped_refptr<media::VideoFrame> frame; 434 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb; 435 const base::TimeTicks start_time = base::TimeTicks::Now(); 436 if (paint_subscriber_.ShouldCaptureFrame(start_time, 437 &frame, 438 &deliver_frame_cb)) { 439 // This message happens just before paint. If we post a task to do the copy, 440 // it should run soon after the paint. 441 BrowserThread::PostTask( 442 BrowserThread::UI, FROM_HERE, 443 base::Bind(capture_callback_, start_time, frame, deliver_frame_cb)); 444 } 445 } 446 447 void ContentCaptureSubscription::OnTimer() { 448 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 449 TRACE_EVENT0("mirroring", "ContentCaptureSubscription::OnTimer"); 450 451 scoped_refptr<media::VideoFrame> frame; 452 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb; 453 454 const base::TimeTicks start_time = base::TimeTicks::Now(); 455 if (timer_subscriber_.ShouldCaptureFrame(start_time, 456 &frame, 457 &deliver_frame_cb)) { 458 capture_callback_.Run(start_time, frame, deliver_frame_cb); 459 } 460 } 461 462 void RenderVideoFrame(const SkBitmap& input, 463 const scoped_refptr<media::VideoFrame>& output, 464 const base::Callback<void(bool)>& done_cb) { 465 base::ScopedClosureRunner failure_handler(base::Bind(done_cb, false)); 466 467 SkAutoLockPixels locker(input); 468 469 // Sanity-check the captured bitmap. 470 if (input.empty() || 471 !input.readyToDraw() || 472 input.config() != SkBitmap::kARGB_8888_Config || 473 input.width() < 2 || input.height() < 2) { 474 DVLOG(1) << "input unacceptable (size=" 475 << input.getSize() 476 << ", ready=" << input.readyToDraw() 477 << ", config=" << input.config() << ')'; 478 return; 479 } 480 481 // Sanity-check the output buffer. 482 if (output->format() != media::VideoFrame::I420) { 483 NOTREACHED(); 484 return; 485 } 486 487 // Calculate the width and height of the content region in the |output|, based 488 // on the aspect ratio of |input|. 489 gfx::Rect region_in_frame = ComputeYV12LetterboxRegion( 490 output->coded_size(), gfx::Size(input.width(), input.height())); 491 492 // Scale the bitmap to the required size, if necessary. 493 SkBitmap scaled_bitmap; 494 if (input.width() != region_in_frame.width() || 495 input.height() != region_in_frame.height()) { 496 497 skia::ImageOperations::ResizeMethod method; 498 if (input.width() < region_in_frame.width() || 499 input.height() < region_in_frame.height()) { 500 // Avoid box filtering when magnifying, because it's actually 501 // nearest-neighbor. 502 method = skia::ImageOperations::RESIZE_HAMMING1; 503 } else { 504 method = skia::ImageOperations::RESIZE_BOX; 505 } 506 507 TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", output.get(), "Scale"); 508 scaled_bitmap = skia::ImageOperations::Resize(input, method, 509 region_in_frame.width(), 510 region_in_frame.height()); 511 } else { 512 scaled_bitmap = input; 513 } 514 515 TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", output.get(), "YUV"); 516 { 517 SkAutoLockPixels scaled_bitmap_locker(scaled_bitmap); 518 519 media::CopyRGBToVideoFrame( 520 reinterpret_cast<uint8*>(scaled_bitmap.getPixels()), 521 scaled_bitmap.rowBytes(), 522 region_in_frame, 523 output.get()); 524 } 525 526 // The result is now ready. 527 ignore_result(failure_handler.Release()); 528 done_cb.Run(true); 529 } 530 531 VideoFrameDeliveryLog::VideoFrameDeliveryLog() 532 : last_frame_rate_log_time_(), 533 count_frames_rendered_(0) { 534 } 535 536 void VideoFrameDeliveryLog::ChronicleFrameDelivery(base::TimeTicks frame_time) { 537 // Log frame rate, if verbose logging is turned on. 538 static const base::TimeDelta kFrameRateLogInterval = 539 base::TimeDelta::FromSeconds(10); 540 if (last_frame_rate_log_time_.is_null()) { 541 last_frame_rate_log_time_ = frame_time; 542 count_frames_rendered_ = 0; 543 } else { 544 ++count_frames_rendered_; 545 const base::TimeDelta elapsed = frame_time - last_frame_rate_log_time_; 546 if (elapsed >= kFrameRateLogInterval) { 547 const double measured_fps = 548 count_frames_rendered_ / elapsed.InSecondsF(); 549 UMA_HISTOGRAM_COUNTS( 550 "TabCapture.FrameRate", 551 static_cast<int>(measured_fps)); 552 VLOG(1) << "Current measured frame rate for " 553 << "WebContentsVideoCaptureDevice is " << measured_fps << " FPS."; 554 last_frame_rate_log_time_ = frame_time; 555 count_frames_rendered_ = 0; 556 } 557 } 558 } 559 560 WebContentsCaptureMachine::WebContentsCaptureMachine(int render_process_id, 561 int render_view_id) 562 : initial_render_process_id_(render_process_id), 563 initial_render_view_id_(render_view_id), 564 fullscreen_widget_id_(MSG_ROUTING_NONE), 565 weak_ptr_factory_(this) {} 566 567 WebContentsCaptureMachine::~WebContentsCaptureMachine() { 568 BrowserThread::PostBlockingPoolTask( 569 FROM_HERE, 570 base::Bind(&DeleteOnWorkerThread, base::Passed(&render_thread_), 571 base::Bind(&base::DoNothing))); 572 } 573 574 bool WebContentsCaptureMachine::Start( 575 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy, 576 const media::VideoCaptureParams& params) { 577 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 578 DCHECK(!started_); 579 580 DCHECK(oracle_proxy.get()); 581 oracle_proxy_ = oracle_proxy; 582 capture_params_ = params; 583 584 render_thread_.reset(new base::Thread("WebContentsVideo_RenderThread")); 585 if (!render_thread_->Start()) { 586 DVLOG(1) << "Failed to spawn render thread."; 587 render_thread_.reset(); 588 return false; 589 } 590 591 if (!StartObservingWebContents()) { 592 DVLOG(1) << "Failed to observe web contents."; 593 render_thread_.reset(); 594 return false; 595 } 596 597 started_ = true; 598 return true; 599 } 600 601 void WebContentsCaptureMachine::Stop(const base::Closure& callback) { 602 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 603 subscription_.reset(); 604 if (web_contents()) { 605 web_contents()->DecrementCapturerCount(); 606 Observe(NULL); 607 } 608 609 // Any callback that intend to use render_thread_ will not work after it is 610 // passed. 611 weak_ptr_factory_.InvalidateWeakPtrs(); 612 613 // The render thread cannot be stopped on the UI thread, so post a message 614 // to the thread pool used for blocking operations. 615 BrowserThread::PostBlockingPoolTask( 616 FROM_HERE, 617 base::Bind(&DeleteOnWorkerThread, base::Passed(&render_thread_), 618 callback)); 619 620 started_ = false; 621 } 622 623 void WebContentsCaptureMachine::Capture( 624 const base::TimeTicks& start_time, 625 const scoped_refptr<media::VideoFrame>& target, 626 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& 627 deliver_frame_cb) { 628 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 629 630 RenderWidgetHost* rwh = GetTarget(); 631 RenderWidgetHostViewBase* view = 632 rwh ? static_cast<RenderWidgetHostViewBase*>(rwh->GetView()) : NULL; 633 if (!view || !rwh) { 634 deliver_frame_cb.Run(base::TimeTicks(), false); 635 return; 636 } 637 638 gfx::Size video_size = target->coded_size(); 639 gfx::Size view_size = view->GetViewBounds().size(); 640 gfx::Size fitted_size; 641 if (!view_size.IsEmpty()) { 642 fitted_size = ComputeYV12LetterboxRegion(video_size, view_size).size(); 643 } 644 if (view_size != last_view_size_) { 645 last_view_size_ = view_size; 646 647 // Measure the number of kilopixels. 648 UMA_HISTOGRAM_COUNTS_10000( 649 "TabCapture.ViewChangeKiloPixels", 650 view_size.width() * view_size.height() / 1024); 651 } 652 653 if (view->CanCopyToVideoFrame()) { 654 view->CopyFromCompositingSurfaceToVideoFrame( 655 gfx::Rect(view_size), 656 target, 657 base::Bind(&WebContentsCaptureMachine:: 658 DidCopyFromCompositingSurfaceToVideoFrame, 659 weak_ptr_factory_.GetWeakPtr(), 660 start_time, deliver_frame_cb)); 661 } else { 662 rwh->CopyFromBackingStore( 663 gfx::Rect(), 664 fitted_size, // Size here is a request not always honored. 665 base::Bind(&WebContentsCaptureMachine::DidCopyFromBackingStore, 666 weak_ptr_factory_.GetWeakPtr(), 667 start_time, 668 target, 669 deliver_frame_cb), 670 SkBitmap::kARGB_8888_Config); 671 } 672 } 673 674 bool WebContentsCaptureMachine::StartObservingWebContents() { 675 // Look-up the RenderViewHost and, from that, the WebContents that wraps it. 676 // If successful, begin observing the WebContents instance. 677 // 678 // Why this can be unsuccessful: The request for mirroring originates in a 679 // render process, and this request is based on the current RenderView 680 // associated with a tab. However, by the time we get up-and-running here, 681 // there have been multiple back-and-forth IPCs between processes, as well as 682 // a bit of indirection across threads. It's easily possible that, in the 683 // meantime, the original RenderView may have gone away. 684 RenderViewHost* const rvh = 685 RenderViewHost::FromID(initial_render_process_id_, 686 initial_render_view_id_); 687 DVLOG_IF(1, !rvh) << "RenderViewHost::FromID(" 688 << initial_render_process_id_ << ", " 689 << initial_render_view_id_ << ") returned NULL."; 690 Observe(rvh ? WebContents::FromRenderViewHost(rvh) : NULL); 691 692 WebContentsImpl* contents = static_cast<WebContentsImpl*>(web_contents()); 693 if (contents) { 694 contents->IncrementCapturerCount(oracle_proxy_->GetCaptureSize()); 695 fullscreen_widget_id_ = contents->GetFullscreenWidgetRoutingID(); 696 RenewFrameSubscription(); 697 return true; 698 } 699 700 DVLOG(1) << "WebContents::FromRenderViewHost(" << rvh << ") returned NULL."; 701 return false; 702 } 703 704 void WebContentsCaptureMachine::WebContentsDestroyed() { 705 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 706 707 subscription_.reset(); 708 web_contents()->DecrementCapturerCount(); 709 oracle_proxy_->ReportError("WebContentsDestroyed()"); 710 } 711 712 RenderWidgetHost* WebContentsCaptureMachine::GetTarget() { 713 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 714 if (!web_contents()) 715 return NULL; 716 717 RenderWidgetHost* rwh = NULL; 718 if (fullscreen_widget_id_ != MSG_ROUTING_NONE) { 719 RenderProcessHost* process = web_contents()->GetRenderProcessHost(); 720 if (process) 721 rwh = RenderWidgetHost::FromID(process->GetID(), fullscreen_widget_id_); 722 } else { 723 rwh = web_contents()->GetRenderViewHost(); 724 } 725 726 return rwh; 727 } 728 729 void WebContentsCaptureMachine::DidCopyFromBackingStore( 730 const base::TimeTicks& start_time, 731 const scoped_refptr<media::VideoFrame>& target, 732 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& 733 deliver_frame_cb, 734 bool success, 735 const SkBitmap& bitmap) { 736 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 737 738 base::TimeTicks now = base::TimeTicks::Now(); 739 DCHECK(render_thread_.get()); 740 if (success) { 741 UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeBitmap", now - start_time); 742 TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", target.get(), 743 "Render"); 744 render_thread_->message_loop_proxy()->PostTask(FROM_HERE, base::Bind( 745 &RenderVideoFrame, bitmap, target, 746 base::Bind(deliver_frame_cb, start_time))); 747 } else { 748 // Capture can fail due to transient issues, so just skip this frame. 749 DVLOG(1) << "CopyFromBackingStore failed; skipping frame."; 750 deliver_frame_cb.Run(start_time, false); 751 } 752 } 753 754 void WebContentsCaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame( 755 const base::TimeTicks& start_time, 756 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& 757 deliver_frame_cb, 758 bool success) { 759 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 760 base::TimeTicks now = base::TimeTicks::Now(); 761 762 if (success) { 763 UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeVideoFrame", now - start_time); 764 } else { 765 // Capture can fail due to transient issues, so just skip this frame. 766 DVLOG(1) << "CopyFromCompositingSurface failed; skipping frame."; 767 } 768 deliver_frame_cb.Run(start_time, success); 769 } 770 771 void WebContentsCaptureMachine::RenewFrameSubscription() { 772 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 773 774 // Always destroy the old subscription before creating a new one. 775 subscription_.reset(); 776 777 RenderWidgetHost* rwh = GetTarget(); 778 if (!rwh || !rwh->GetView()) 779 return; 780 781 subscription_.reset(new ContentCaptureSubscription(*rwh, oracle_proxy_, 782 base::Bind(&WebContentsCaptureMachine::Capture, 783 weak_ptr_factory_.GetWeakPtr()))); 784 } 785 786 } // namespace 787 788 WebContentsVideoCaptureDevice::WebContentsVideoCaptureDevice( 789 int render_process_id, int render_view_id) 790 : core_(new ContentVideoCaptureDeviceCore(scoped_ptr<VideoCaptureMachine>( 791 new WebContentsCaptureMachine(render_process_id, render_view_id)))) {} 792 793 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() { 794 DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying."; 795 } 796 797 // static 798 media::VideoCaptureDevice* WebContentsVideoCaptureDevice::Create( 799 const std::string& device_id) { 800 // Parse device_id into render_process_id and render_view_id. 801 int render_process_id = -1; 802 int render_view_id = -1; 803 if (!WebContentsCaptureUtil::ExtractTabCaptureTarget( 804 device_id, &render_process_id, &render_view_id)) { 805 return NULL; 806 } 807 808 return new WebContentsVideoCaptureDevice(render_process_id, render_view_id); 809 } 810 811 void WebContentsVideoCaptureDevice::AllocateAndStart( 812 const media::VideoCaptureParams& params, 813 scoped_ptr<Client> client) { 814 DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString(); 815 core_->AllocateAndStart(params, client.Pass()); 816 } 817 818 void WebContentsVideoCaptureDevice::StopAndDeAllocate() { 819 core_->StopAndDeAllocate(); 820 } 821 822 } // namespace content 823