1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "content/browser/renderer_host/media/video_capture_controller.h" 6 7 #include <set> 8 9 #include "base/bind.h" 10 #include "base/debug/trace_event.h" 11 #include "base/stl_util.h" 12 #include "content/browser/renderer_host/media/media_stream_manager.h" 13 #include "content/browser/renderer_host/media/video_capture_manager.h" 14 #include "content/public/browser/browser_thread.h" 15 #include "media/base/video_frame.h" 16 #include "media/base/video_util.h" 17 #include "media/base/yuv_convert.h" 18 19 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW) 20 #include "third_party/libyuv/include/libyuv.h" 21 #endif 22 23 using media::VideoCaptureFormat; 24 25 namespace content { 26 27 namespace { 28 29 // The number of buffers that VideoCaptureBufferPool should allocate. 30 const int kNoOfBuffers = 3; 31 32 class PoolBuffer : public media::VideoCaptureDevice::Client::Buffer { 33 public: 34 PoolBuffer(const scoped_refptr<VideoCaptureBufferPool>& pool, 35 int buffer_id, 36 void* data, 37 size_t size) 38 : Buffer(buffer_id, data, size), pool_(pool) { 39 DCHECK(pool_); 40 } 41 42 private: 43 virtual ~PoolBuffer() { pool_->RelinquishProducerReservation(id()); } 44 45 const scoped_refptr<VideoCaptureBufferPool> pool_; 46 }; 47 48 } // anonymous namespace 49 50 struct VideoCaptureController::ControllerClient { 51 ControllerClient(const VideoCaptureControllerID& id, 52 VideoCaptureControllerEventHandler* handler, 53 base::ProcessHandle render_process, 54 media::VideoCaptureSessionId session_id, 55 const media::VideoCaptureParams& params) 56 : controller_id(id), 57 event_handler(handler), 58 render_process_handle(render_process), 59 session_id(session_id), 60 parameters(params), 61 session_closed(false) {} 62 63 ~ControllerClient() {} 64 65 // ID used for identifying this object. 66 const VideoCaptureControllerID controller_id; 67 VideoCaptureControllerEventHandler* const event_handler; 68 69 // Handle to the render process that will receive the capture buffers. 70 const base::ProcessHandle render_process_handle; 71 const media::VideoCaptureSessionId session_id; 72 const media::VideoCaptureParams parameters; 73 74 // Buffers that are currently known to this client. 75 std::set<int> known_buffers; 76 77 // Buffers currently held by this client. 78 std::set<int> active_buffers; 79 80 // State of capture session, controlled by VideoCaptureManager directly. This 81 // transitions to true as soon as StopSession() occurs, at which point the 82 // client is sent an OnEnded() event. However, because the client retains a 83 // VideoCaptureController* pointer, its ControllerClient entry lives on until 84 // it unregisters itself via RemoveClient(), which may happen asynchronously. 85 // 86 // TODO(nick): If we changed the semantics of VideoCaptureHost so that 87 // OnEnded() events were processed synchronously (with the RemoveClient() done 88 // implicitly), we could avoid tracking this state here in the Controller, and 89 // simplify the code in both places. 90 bool session_closed; 91 }; 92 93 // Receives events from the VideoCaptureDevice and posts them to a 94 // VideoCaptureController on the IO thread. An instance of this class may safely 95 // outlive its target VideoCaptureController. 96 // 97 // Methods of this class may be called from any thread, and in practice will 98 // often be called on some auxiliary thread depending on the platform and the 99 // device type; including, for example, the DirectShow thread on Windows, the 100 // v4l2_thread on Linux, and the UI thread for tab capture. 101 class VideoCaptureController::VideoCaptureDeviceClient 102 : public media::VideoCaptureDevice::Client { 103 public: 104 explicit VideoCaptureDeviceClient( 105 const base::WeakPtr<VideoCaptureController>& controller, 106 const scoped_refptr<VideoCaptureBufferPool>& buffer_pool); 107 virtual ~VideoCaptureDeviceClient(); 108 109 // VideoCaptureDevice::Client implementation. 110 virtual scoped_refptr<Buffer> ReserveOutputBuffer( 111 media::VideoFrame::Format format, 112 const gfx::Size& size) OVERRIDE; 113 virtual void OnIncomingCapturedFrame(const uint8* data, 114 int length, 115 base::Time timestamp, 116 int rotation, 117 const VideoCaptureFormat& frame_format) 118 OVERRIDE; 119 virtual void OnIncomingCapturedBuffer(const scoped_refptr<Buffer>& buffer, 120 media::VideoFrame::Format format, 121 const gfx::Size& dimensions, 122 base::Time timestamp, 123 int frame_rate) OVERRIDE; 124 virtual void OnError() OVERRIDE; 125 126 private: 127 scoped_refptr<Buffer> DoReserveOutputBuffer(media::VideoFrame::Format format, 128 const gfx::Size& dimensions); 129 130 // The controller to which we post events. 131 const base::WeakPtr<VideoCaptureController> controller_; 132 133 // The pool of shared-memory buffers used for capturing. 134 const scoped_refptr<VideoCaptureBufferPool> buffer_pool_; 135 }; 136 137 VideoCaptureController::VideoCaptureController() 138 : buffer_pool_(new VideoCaptureBufferPool(kNoOfBuffers)), 139 state_(VIDEO_CAPTURE_STATE_STARTED), 140 weak_ptr_factory_(this) { 141 } 142 143 VideoCaptureController::VideoCaptureDeviceClient::VideoCaptureDeviceClient( 144 const base::WeakPtr<VideoCaptureController>& controller, 145 const scoped_refptr<VideoCaptureBufferPool>& buffer_pool) 146 : controller_(controller), buffer_pool_(buffer_pool) {} 147 148 VideoCaptureController::VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {} 149 150 base::WeakPtr<VideoCaptureController> VideoCaptureController::GetWeakPtr() { 151 return weak_ptr_factory_.GetWeakPtr(); 152 } 153 154 scoped_ptr<media::VideoCaptureDevice::Client> 155 VideoCaptureController::NewDeviceClient() { 156 scoped_ptr<media::VideoCaptureDevice::Client> result( 157 new VideoCaptureDeviceClient(this->GetWeakPtr(), buffer_pool_)); 158 return result.Pass(); 159 } 160 161 void VideoCaptureController::AddClient( 162 const VideoCaptureControllerID& id, 163 VideoCaptureControllerEventHandler* event_handler, 164 base::ProcessHandle render_process, 165 media::VideoCaptureSessionId session_id, 166 const media::VideoCaptureParams& params) { 167 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 168 DVLOG(1) << "VideoCaptureController::AddClient, id " << id.device_id 169 << ", " << params.requested_format.frame_size.ToString() 170 << ", " << params.requested_format.frame_rate 171 << ", " << session_id 172 << ")"; 173 174 // If this is the first client added to the controller, cache the parameters. 175 if (!controller_clients_.size()) 176 video_capture_format_ = params.requested_format; 177 178 // Signal error in case device is already in error state. 179 if (state_ == VIDEO_CAPTURE_STATE_ERROR) { 180 event_handler->OnError(id); 181 return; 182 } 183 184 // Do nothing if this client has called AddClient before. 185 if (FindClient(id, event_handler, controller_clients_)) 186 return; 187 188 ControllerClient* client = new ControllerClient( 189 id, event_handler, render_process, session_id, params); 190 // If we already have gotten frame_info from the device, repeat it to the new 191 // client. 192 if (state_ == VIDEO_CAPTURE_STATE_STARTED) { 193 controller_clients_.push_back(client); 194 return; 195 } 196 } 197 198 int VideoCaptureController::RemoveClient( 199 const VideoCaptureControllerID& id, 200 VideoCaptureControllerEventHandler* event_handler) { 201 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 202 DVLOG(1) << "VideoCaptureController::RemoveClient, id " << id.device_id; 203 204 ControllerClient* client = FindClient(id, event_handler, controller_clients_); 205 if (!client) 206 return kInvalidMediaCaptureSessionId; 207 208 // Take back all buffers held by the |client|. 209 for (std::set<int>::iterator buffer_it = client->active_buffers.begin(); 210 buffer_it != client->active_buffers.end(); 211 ++buffer_it) { 212 int buffer_id = *buffer_it; 213 buffer_pool_->RelinquishConsumerHold(buffer_id, 1); 214 } 215 client->active_buffers.clear(); 216 217 int session_id = client->session_id; 218 controller_clients_.remove(client); 219 delete client; 220 221 return session_id; 222 } 223 224 void VideoCaptureController::StopSession(int session_id) { 225 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 226 DVLOG(1) << "VideoCaptureController::StopSession, id " << session_id; 227 228 ControllerClient* client = FindClient(session_id, controller_clients_); 229 230 if (client) { 231 client->session_closed = true; 232 client->event_handler->OnEnded(client->controller_id); 233 } 234 } 235 236 void VideoCaptureController::ReturnBuffer( 237 const VideoCaptureControllerID& id, 238 VideoCaptureControllerEventHandler* event_handler, 239 int buffer_id) { 240 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 241 242 ControllerClient* client = FindClient(id, event_handler, controller_clients_); 243 244 // If this buffer is not held by this client, or this client doesn't exist 245 // in controller, do nothing. 246 if (!client || !client->active_buffers.erase(buffer_id)) { 247 NOTREACHED(); 248 return; 249 } 250 251 buffer_pool_->RelinquishConsumerHold(buffer_id, 1); 252 } 253 254 const media::VideoCaptureFormat& 255 VideoCaptureController::GetVideoCaptureFormat() const { 256 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 257 return video_capture_format_; 258 } 259 260 scoped_refptr<media::VideoCaptureDevice::Client::Buffer> 261 VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer( 262 media::VideoFrame::Format format, 263 const gfx::Size& size) { 264 return DoReserveOutputBuffer(format, size); 265 } 266 267 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame( 268 const uint8* data, 269 int length, 270 base::Time timestamp, 271 int rotation, 272 const VideoCaptureFormat& frame_format) { 273 TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame"); 274 275 if (!frame_format.IsValid()) 276 return; 277 278 // Chopped pixels in width/height in case video capture device has odd 279 // numbers for width/height. 280 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW) 281 int chopped_width = 0; 282 int chopped_height = 0; 283 #endif 284 int new_unrotated_width = frame_format.frame_size.width(); 285 int new_unrotated_height = frame_format.frame_size.height(); 286 287 if (new_unrotated_width & 1) { 288 --new_unrotated_width; 289 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW) 290 chopped_width = 1; 291 #endif 292 } 293 if (new_unrotated_height & 1) { 294 --new_unrotated_height; 295 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW) 296 chopped_height = 1; 297 #endif 298 } 299 300 int destination_width = new_unrotated_width; 301 int destination_height = new_unrotated_height; 302 if (rotation == 90 || rotation == 270) { 303 destination_width = new_unrotated_height; 304 destination_height = new_unrotated_width; 305 } 306 const gfx::Size dimensions(destination_width, destination_height); 307 scoped_refptr<Buffer> buffer = 308 DoReserveOutputBuffer(media::VideoFrame::I420, dimensions); 309 310 if (!buffer) 311 return; 312 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW) 313 uint8* yplane = reinterpret_cast<uint8*>(buffer->data()); 314 uint8* uplane = 315 yplane + 316 media::VideoFrame::PlaneAllocationSize( 317 media::VideoFrame::I420, media::VideoFrame::kYPlane, dimensions); 318 uint8* vplane = 319 uplane + 320 media::VideoFrame::PlaneAllocationSize( 321 media::VideoFrame::I420, media::VideoFrame::kUPlane, dimensions); 322 int yplane_stride = dimensions.width(); 323 int uv_plane_stride = yplane_stride / 2; 324 int crop_x = 0; 325 int crop_y = 0; 326 libyuv::FourCC origin_colorspace = libyuv::FOURCC_ANY; 327 328 libyuv::RotationMode rotation_mode = libyuv::kRotate0; 329 if (rotation == 90) 330 rotation_mode = libyuv::kRotate90; 331 else if (rotation == 180) 332 rotation_mode = libyuv::kRotate180; 333 else if (rotation == 270) 334 rotation_mode = libyuv::kRotate270; 335 336 switch (frame_format.pixel_format) { 337 case media::PIXEL_FORMAT_UNKNOWN: // Color format not set. 338 break; 339 case media::PIXEL_FORMAT_I420: 340 DCHECK(!chopped_width && !chopped_height); 341 origin_colorspace = libyuv::FOURCC_I420; 342 break; 343 case media::PIXEL_FORMAT_YV12: 344 DCHECK(!chopped_width && !chopped_height); 345 origin_colorspace = libyuv::FOURCC_YV12; 346 break; 347 case media::PIXEL_FORMAT_NV21: 348 DCHECK(!chopped_width && !chopped_height); 349 origin_colorspace = libyuv::FOURCC_NV21; 350 break; 351 case media::PIXEL_FORMAT_YUY2: 352 DCHECK(!chopped_width && !chopped_height); 353 origin_colorspace = libyuv::FOURCC_YUY2; 354 break; 355 case media::PIXEL_FORMAT_UYVY: 356 DCHECK(!chopped_width && !chopped_height); 357 origin_colorspace = libyuv::FOURCC_UYVY; 358 break; 359 case media::PIXEL_FORMAT_RGB24: 360 origin_colorspace = libyuv::FOURCC_RAW; 361 break; 362 case media::PIXEL_FORMAT_ARGB: 363 origin_colorspace = libyuv::FOURCC_ARGB; 364 break; 365 case media::PIXEL_FORMAT_MJPEG: 366 origin_colorspace = libyuv::FOURCC_MJPG; 367 break; 368 default: 369 NOTREACHED(); 370 } 371 372 int need_convert_rgb24_on_win = false; 373 #if defined(OS_WIN) 374 // TODO(wjia): Use libyuv::ConvertToI420 since support for image inversion 375 // (vertical flipping) has been added. Use negative src_height as indicator. 376 if (frame_format.pixel_format == media::PIXEL_FORMAT_RGB24) { 377 // Rotation is not supported in kRGB24 and OS_WIN case. 378 DCHECK(!rotation); 379 need_convert_rgb24_on_win = true; 380 } 381 #endif 382 if (need_convert_rgb24_on_win) { 383 int rgb_stride = -3 * (new_unrotated_width + chopped_width); 384 const uint8* rgb_src = 385 data + 3 * (new_unrotated_width + chopped_width) * 386 (new_unrotated_height - 1 + chopped_height); 387 media::ConvertRGB24ToYUV(rgb_src, 388 yplane, 389 uplane, 390 vplane, 391 new_unrotated_width, 392 new_unrotated_height, 393 rgb_stride, 394 yplane_stride, 395 uv_plane_stride); 396 } else { 397 libyuv::ConvertToI420(data, 398 length, 399 yplane, 400 yplane_stride, 401 uplane, 402 uv_plane_stride, 403 vplane, 404 uv_plane_stride, 405 crop_x, 406 crop_y, 407 new_unrotated_width + chopped_width, 408 new_unrotated_height, 409 new_unrotated_width, 410 new_unrotated_height, 411 rotation_mode, 412 origin_colorspace); 413 } 414 #else 415 // Libyuv is not linked in for Android WebView builds, but video capture is 416 // not used in those builds either. Whenever libyuv is added in that build, 417 // address all these #ifdef parts, see http://crbug.com/299611 . 418 NOTREACHED(); 419 #endif // if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW) 420 BrowserThread::PostTask( 421 BrowserThread::IO, 422 FROM_HERE, 423 base::Bind( 424 &VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread, 425 controller_, 426 buffer, 427 dimensions, 428 frame_format.frame_rate, 429 timestamp)); 430 } 431 432 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedBuffer( 433 const scoped_refptr<Buffer>& buffer, 434 media::VideoFrame::Format format, 435 const gfx::Size& dimensions, 436 base::Time timestamp, 437 int frame_rate) { 438 // The capture pipeline expects I420 for now. 439 DCHECK_EQ(format, media::VideoFrame::I420) 440 << "Non-I420 output buffer returned"; 441 442 BrowserThread::PostTask( 443 BrowserThread::IO, 444 FROM_HERE, 445 base::Bind( 446 &VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread, 447 controller_, 448 buffer, 449 dimensions, 450 frame_rate, 451 timestamp)); 452 } 453 454 void VideoCaptureController::VideoCaptureDeviceClient::OnError() { 455 BrowserThread::PostTask(BrowserThread::IO, 456 FROM_HERE, 457 base::Bind(&VideoCaptureController::DoErrorOnIOThread, controller_)); 458 } 459 460 scoped_refptr<media::VideoCaptureDevice::Client::Buffer> 461 VideoCaptureController::VideoCaptureDeviceClient::DoReserveOutputBuffer( 462 media::VideoFrame::Format format, 463 const gfx::Size& dimensions) { 464 // The capture pipeline expects I420 for now. 465 DCHECK_EQ(format, media::VideoFrame::I420) 466 << "Non-I420 output buffer requested"; 467 468 int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId; 469 const size_t frame_bytes = 470 media::VideoFrame::AllocationSize(format, dimensions); 471 472 int buffer_id = 473 buffer_pool_->ReserveForProducer(frame_bytes, &buffer_id_to_drop); 474 if (buffer_id == VideoCaptureBufferPool::kInvalidId) 475 return NULL; 476 void* data; 477 size_t size; 478 buffer_pool_->GetBufferInfo(buffer_id, &data, &size); 479 480 scoped_refptr<media::VideoCaptureDevice::Client::Buffer> output_buffer( 481 new PoolBuffer(buffer_pool_, buffer_id, data, size)); 482 483 if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) { 484 BrowserThread::PostTask(BrowserThread::IO, 485 FROM_HERE, 486 base::Bind(&VideoCaptureController::DoBufferDestroyedOnIOThread, 487 controller_, buffer_id_to_drop)); 488 } 489 490 return output_buffer; 491 } 492 493 VideoCaptureController::~VideoCaptureController() { 494 STLDeleteContainerPointers(controller_clients_.begin(), 495 controller_clients_.end()); 496 } 497 498 void VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread( 499 scoped_refptr<media::VideoCaptureDevice::Client::Buffer> buffer, 500 const gfx::Size& dimensions, 501 int frame_rate, 502 base::Time timestamp) { 503 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 504 DCHECK_NE(buffer->id(), VideoCaptureBufferPool::kInvalidId); 505 506 VideoCaptureFormat frame_format( 507 dimensions, frame_rate, media::PIXEL_FORMAT_I420); 508 509 int count = 0; 510 if (state_ == VIDEO_CAPTURE_STATE_STARTED) { 511 for (ControllerClients::iterator client_it = controller_clients_.begin(); 512 client_it != controller_clients_.end(); ++client_it) { 513 ControllerClient* client = *client_it; 514 if (client->session_closed) 515 continue; 516 517 bool is_new_buffer = client->known_buffers.insert(buffer->id()).second; 518 if (is_new_buffer) { 519 // On the first use of a buffer on a client, share the memory handle. 520 size_t memory_size = 0; 521 base::SharedMemoryHandle remote_handle = buffer_pool_->ShareToProcess( 522 buffer->id(), client->render_process_handle, &memory_size); 523 client->event_handler->OnBufferCreated( 524 client->controller_id, remote_handle, memory_size, buffer->id()); 525 } 526 527 client->event_handler->OnBufferReady( 528 client->controller_id, buffer->id(), timestamp, frame_format); 529 bool inserted = client->active_buffers.insert(buffer->id()).second; 530 DCHECK(inserted) << "Unexpected duplicate buffer: " << buffer->id(); 531 count++; 532 } 533 } 534 535 buffer_pool_->HoldForConsumers(buffer->id(), count); 536 } 537 538 void VideoCaptureController::DoErrorOnIOThread() { 539 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 540 state_ = VIDEO_CAPTURE_STATE_ERROR; 541 542 for (ControllerClients::iterator client_it = controller_clients_.begin(); 543 client_it != controller_clients_.end(); ++client_it) { 544 ControllerClient* client = *client_it; 545 if (client->session_closed) 546 continue; 547 548 client->event_handler->OnError(client->controller_id); 549 } 550 } 551 552 void VideoCaptureController::DoBufferDestroyedOnIOThread( 553 int buffer_id_to_drop) { 554 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 555 556 for (ControllerClients::iterator client_it = controller_clients_.begin(); 557 client_it != controller_clients_.end(); ++client_it) { 558 ControllerClient* client = *client_it; 559 if (client->session_closed) 560 continue; 561 562 if (client->known_buffers.erase(buffer_id_to_drop)) { 563 client->event_handler->OnBufferDestroyed(client->controller_id, 564 buffer_id_to_drop); 565 } 566 } 567 } 568 569 VideoCaptureController::ControllerClient* 570 VideoCaptureController::FindClient( 571 const VideoCaptureControllerID& id, 572 VideoCaptureControllerEventHandler* handler, 573 const ControllerClients& clients) { 574 for (ControllerClients::const_iterator client_it = clients.begin(); 575 client_it != clients.end(); ++client_it) { 576 if ((*client_it)->controller_id == id && 577 (*client_it)->event_handler == handler) { 578 return *client_it; 579 } 580 } 581 return NULL; 582 } 583 584 VideoCaptureController::ControllerClient* 585 VideoCaptureController::FindClient( 586 int session_id, 587 const ControllerClients& clients) { 588 for (ControllerClients::const_iterator client_it = clients.begin(); 589 client_it != clients.end(); ++client_it) { 590 if ((*client_it)->session_id == session_id) { 591 return *client_it; 592 } 593 } 594 return NULL; 595 } 596 597 int VideoCaptureController::GetClientCount() { 598 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 599 return controller_clients_.size(); 600 } 601 602 } // namespace content 603