1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 // 5 // Notes about usage of this object by VideoCaptureImplManager. 6 // 7 // VideoCaptureImplManager access this object by using a Unretained() 8 // binding and tasks on the IO thread. It is then important that 9 // VideoCaptureImpl never post task to itself. All operations must be 10 // synchronous. 11 12 #include "content/renderer/media/video_capture_impl.h" 13 14 #include "base/bind.h" 15 #include "base/stl_util.h" 16 #include "content/child/child_process.h" 17 #include "content/common/media/video_capture_messages.h" 18 #include "media/base/bind_to_current_loop.h" 19 #include "media/base/limits.h" 20 #include "media/base/video_frame.h" 21 22 namespace content { 23 24 class VideoCaptureImpl::ClientBuffer 25 : public base::RefCountedThreadSafe<ClientBuffer> { 26 public: 27 ClientBuffer(scoped_ptr<base::SharedMemory> buffer, 28 size_t buffer_size) 29 : buffer(buffer.Pass()), 30 buffer_size(buffer_size) {} 31 const scoped_ptr<base::SharedMemory> buffer; 32 const size_t buffer_size; 33 34 private: 35 friend class base::RefCountedThreadSafe<ClientBuffer>; 36 37 virtual ~ClientBuffer() {} 38 39 DISALLOW_COPY_AND_ASSIGN(ClientBuffer); 40 }; 41 42 VideoCaptureImpl::ClientInfo::ClientInfo() {} 43 VideoCaptureImpl::ClientInfo::~ClientInfo() {} 44 45 VideoCaptureImpl::VideoCaptureImpl( 46 const media::VideoCaptureSessionId session_id, 47 VideoCaptureMessageFilter* filter) 48 : message_filter_(filter), 49 device_id_(0), 50 session_id_(session_id), 51 suspended_(false), 52 state_(VIDEO_CAPTURE_STATE_STOPPED), 53 weak_factory_(this) { 54 DCHECK(filter); 55 thread_checker_.DetachFromThread(); 56 } 57 58 VideoCaptureImpl::~VideoCaptureImpl() { 59 DCHECK(thread_checker_.CalledOnValidThread()); 60 } 61 62 void VideoCaptureImpl::Init() { 63 DCHECK(thread_checker_.CalledOnValidThread()); 64 message_filter_->AddDelegate(this); 65 } 66 67 void VideoCaptureImpl::DeInit() { 68 DCHECK(thread_checker_.CalledOnValidThread()); 69 if (state_ == VIDEO_CAPTURE_STATE_STARTED) 70 Send(new VideoCaptureHostMsg_Stop(device_id_)); 71 message_filter_->RemoveDelegate(this); 72 } 73 74 void VideoCaptureImpl::SuspendCapture(bool suspend) { 75 DCHECK(thread_checker_.CalledOnValidThread()); 76 Send(suspend ? 77 static_cast<IPC::Message*>(new VideoCaptureHostMsg_Pause(device_id_)) : 78 static_cast<IPC::Message*>( 79 new VideoCaptureHostMsg_Resume(device_id_, session_id_, params_))); 80 } 81 82 void VideoCaptureImpl::StartCapture( 83 int client_id, 84 const media::VideoCaptureParams& params, 85 const VideoCaptureStateUpdateCB& state_update_cb, 86 const VideoCaptureDeliverFrameCB& deliver_frame_cb) { 87 DCHECK(thread_checker_.CalledOnValidThread()); 88 ClientInfo client_info; 89 client_info.params = params; 90 client_info.state_update_cb = state_update_cb; 91 client_info.deliver_frame_cb = deliver_frame_cb; 92 93 if (state_ == VIDEO_CAPTURE_STATE_ERROR) { 94 state_update_cb.Run(VIDEO_CAPTURE_STATE_ERROR); 95 } else if (clients_pending_on_filter_.count(client_id) || 96 clients_pending_on_restart_.count(client_id) || 97 clients_.count(client_id)) { 98 LOG(FATAL) << "This client has already started."; 99 } else if (!device_id_) { 100 clients_pending_on_filter_[client_id] = client_info; 101 } else { 102 // Note: |state_| might not be started at this point. But we tell 103 // client that we have started. 104 state_update_cb.Run(VIDEO_CAPTURE_STATE_STARTED); 105 if (state_ == VIDEO_CAPTURE_STATE_STARTED) { 106 clients_[client_id] = client_info; 107 // TODO(sheu): Allowing resolution change will require that all 108 // outstanding clients of a capture session support resolution change. 109 DCHECK_EQ(params_.resolution_change_policy, 110 params.resolution_change_policy); 111 } else if (state_ == VIDEO_CAPTURE_STATE_STOPPING) { 112 clients_pending_on_restart_[client_id] = client_info; 113 DVLOG(1) << "StartCapture: Got new resolution " 114 << params.requested_format.frame_size.ToString() 115 << " during stopping."; 116 } else { 117 clients_[client_id] = client_info; 118 if (state_ == VIDEO_CAPTURE_STATE_STARTED) 119 return; 120 params_ = params; 121 if (params_.requested_format.frame_rate > 122 media::limits::kMaxFramesPerSecond) { 123 params_.requested_format.frame_rate = 124 media::limits::kMaxFramesPerSecond; 125 } 126 DVLOG(1) << "StartCapture: starting with first resolution " 127 << params_.requested_format.frame_size.ToString(); 128 first_frame_timestamp_ = base::TimeTicks(); 129 StartCaptureInternal(); 130 } 131 } 132 } 133 134 void VideoCaptureImpl::StopCapture(int client_id) { 135 DCHECK(thread_checker_.CalledOnValidThread()); 136 137 // A client ID can be in only one client list. 138 // If this ID is in any client list, we can just remove it from 139 // that client list and don't have to run the other following RemoveClient(). 140 if (!RemoveClient(client_id, &clients_pending_on_filter_)) { 141 if (!RemoveClient(client_id, &clients_pending_on_restart_)) { 142 RemoveClient(client_id, &clients_); 143 } 144 } 145 146 if (clients_.empty()) { 147 DVLOG(1) << "StopCapture: No more client, stopping ..."; 148 StopDevice(); 149 client_buffers_.clear(); 150 weak_factory_.InvalidateWeakPtrs(); 151 } 152 } 153 154 void VideoCaptureImpl::GetDeviceSupportedFormats( 155 const VideoCaptureDeviceFormatsCB& callback) { 156 DCHECK(thread_checker_.CalledOnValidThread()); 157 device_formats_cb_queue_.push_back(callback); 158 if (device_formats_cb_queue_.size() == 1) 159 Send(new VideoCaptureHostMsg_GetDeviceSupportedFormats(device_id_, 160 session_id_)); 161 } 162 163 void VideoCaptureImpl::GetDeviceFormatsInUse( 164 const VideoCaptureDeviceFormatsCB& callback) { 165 DCHECK(thread_checker_.CalledOnValidThread()); 166 device_formats_in_use_cb_queue_.push_back(callback); 167 if (device_formats_in_use_cb_queue_.size() == 1) 168 Send( 169 new VideoCaptureHostMsg_GetDeviceFormatsInUse(device_id_, session_id_)); 170 } 171 172 void VideoCaptureImpl::OnBufferCreated( 173 base::SharedMemoryHandle handle, 174 int length, int buffer_id) { 175 DCHECK(thread_checker_.CalledOnValidThread()); 176 177 // In case client calls StopCapture before the arrival of created buffer, 178 // just close this buffer and return. 179 if (state_ != VIDEO_CAPTURE_STATE_STARTED) { 180 base::SharedMemory::CloseHandle(handle); 181 return; 182 } 183 184 scoped_ptr<base::SharedMemory> shm(new base::SharedMemory(handle, false)); 185 if (!shm->Map(length)) { 186 DLOG(ERROR) << "OnBufferCreated: Map failed."; 187 return; 188 } 189 190 bool inserted = 191 client_buffers_.insert(std::make_pair( 192 buffer_id, 193 new ClientBuffer(shm.Pass(), 194 length))).second; 195 DCHECK(inserted); 196 } 197 198 void VideoCaptureImpl::OnBufferDestroyed(int buffer_id) { 199 DCHECK(thread_checker_.CalledOnValidThread()); 200 201 ClientBufferMap::iterator iter = client_buffers_.find(buffer_id); 202 if (iter == client_buffers_.end()) 203 return; 204 205 DCHECK(!iter->second.get() || iter->second->HasOneRef()) 206 << "Instructed to delete buffer we are still using."; 207 client_buffers_.erase(iter); 208 } 209 210 void VideoCaptureImpl::OnBufferReceived(int buffer_id, 211 const media::VideoCaptureFormat& format, 212 const gfx::Rect& visible_rect, 213 base::TimeTicks timestamp) { 214 DCHECK(thread_checker_.CalledOnValidThread()); 215 216 // The capture pipeline supports only I420 for now. 217 DCHECK_EQ(format.pixel_format, media::PIXEL_FORMAT_I420); 218 219 if (state_ != VIDEO_CAPTURE_STATE_STARTED || suspended_) { 220 Send(new VideoCaptureHostMsg_BufferReady(device_id_, buffer_id, 0)); 221 return; 222 } 223 224 last_frame_format_ = format; 225 if (first_frame_timestamp_.is_null()) 226 first_frame_timestamp_ = timestamp; 227 228 // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc 229 TRACE_EVENT_INSTANT2( 230 "cast_perf_test", "OnBufferReceived", 231 TRACE_EVENT_SCOPE_THREAD, 232 "timestamp", timestamp.ToInternalValue(), 233 "time_delta", (timestamp - first_frame_timestamp_).ToInternalValue()); 234 235 ClientBufferMap::iterator iter = client_buffers_.find(buffer_id); 236 DCHECK(iter != client_buffers_.end()); 237 scoped_refptr<ClientBuffer> buffer = iter->second; 238 scoped_refptr<media::VideoFrame> frame = 239 media::VideoFrame::WrapExternalPackedMemory( 240 media::VideoFrame::I420, 241 last_frame_format_.frame_size, 242 visible_rect, 243 gfx::Size(visible_rect.width(), visible_rect.height()), 244 reinterpret_cast<uint8*>(buffer->buffer->memory()), 245 buffer->buffer_size, 246 buffer->buffer->handle(), 247 timestamp - first_frame_timestamp_, 248 media::BindToCurrentLoop( 249 base::Bind(&VideoCaptureImpl::OnClientBufferFinished, 250 weak_factory_.GetWeakPtr(), 251 buffer_id, 252 buffer, 253 0))); 254 255 for (ClientInfoMap::iterator it = clients_.begin(); it != clients_.end(); 256 ++it) { 257 it->second.deliver_frame_cb.Run(frame, format, timestamp); 258 } 259 } 260 261 static void NullReadPixelsCB(const SkBitmap& bitmap) { NOTIMPLEMENTED(); } 262 263 void VideoCaptureImpl::OnMailboxBufferReceived( 264 int buffer_id, 265 const gpu::MailboxHolder& mailbox_holder, 266 const media::VideoCaptureFormat& format, 267 base::TimeTicks timestamp) { 268 DCHECK(thread_checker_.CalledOnValidThread()); 269 270 if (state_ != VIDEO_CAPTURE_STATE_STARTED || suspended_) { 271 Send(new VideoCaptureHostMsg_BufferReady(device_id_, buffer_id, 0)); 272 return; 273 } 274 275 last_frame_format_ = format; 276 if (first_frame_timestamp_.is_null()) 277 first_frame_timestamp_ = timestamp; 278 279 scoped_refptr<media::VideoFrame> frame = media::VideoFrame::WrapNativeTexture( 280 make_scoped_ptr(new gpu::MailboxHolder(mailbox_holder)), 281 media::BindToCurrentLoop( 282 base::Bind(&VideoCaptureImpl::OnClientBufferFinished, 283 weak_factory_.GetWeakPtr(), 284 buffer_id, 285 scoped_refptr<ClientBuffer>())), 286 last_frame_format_.frame_size, 287 gfx::Rect(last_frame_format_.frame_size), 288 last_frame_format_.frame_size, 289 timestamp - first_frame_timestamp_, 290 base::Bind(&NullReadPixelsCB)); 291 292 for (ClientInfoMap::iterator it = clients_.begin(); it != clients_.end(); 293 ++it) { 294 it->second.deliver_frame_cb.Run(frame, format, timestamp); 295 } 296 } 297 298 void VideoCaptureImpl::OnClientBufferFinished( 299 int buffer_id, 300 const scoped_refptr<ClientBuffer>& /* ignored_buffer */, 301 uint32 release_sync_point) { 302 DCHECK(thread_checker_.CalledOnValidThread()); 303 Send(new VideoCaptureHostMsg_BufferReady( 304 device_id_, buffer_id, release_sync_point)); 305 } 306 307 void VideoCaptureImpl::OnStateChanged(VideoCaptureState state) { 308 DCHECK(thread_checker_.CalledOnValidThread()); 309 310 switch (state) { 311 case VIDEO_CAPTURE_STATE_STARTED: 312 // Camera has started in the browser process. Since we have already 313 // told all clients that we have started there's nothing to do. 314 break; 315 case VIDEO_CAPTURE_STATE_STOPPED: 316 state_ = VIDEO_CAPTURE_STATE_STOPPED; 317 DVLOG(1) << "OnStateChanged: stopped!, device_id = " << device_id_; 318 client_buffers_.clear(); 319 weak_factory_.InvalidateWeakPtrs(); 320 if (!clients_.empty() || !clients_pending_on_restart_.empty()) 321 RestartCapture(); 322 break; 323 case VIDEO_CAPTURE_STATE_PAUSED: 324 for (ClientInfoMap::iterator it = clients_.begin(); 325 it != clients_.end(); ++it) { 326 it->second.state_update_cb.Run(VIDEO_CAPTURE_STATE_PAUSED); 327 } 328 break; 329 case VIDEO_CAPTURE_STATE_ERROR: 330 DVLOG(1) << "OnStateChanged: error!, device_id = " << device_id_; 331 for (ClientInfoMap::iterator it = clients_.begin(); 332 it != clients_.end(); ++it) { 333 it->second.state_update_cb.Run(VIDEO_CAPTURE_STATE_ERROR); 334 } 335 clients_.clear(); 336 state_ = VIDEO_CAPTURE_STATE_ERROR; 337 break; 338 case VIDEO_CAPTURE_STATE_ENDED: 339 DVLOG(1) << "OnStateChanged: ended!, device_id = " << device_id_; 340 for (ClientInfoMap::iterator it = clients_.begin(); 341 it != clients_.end(); ++it) { 342 // We'll only notify the client that the stream has stopped. 343 it->second.state_update_cb.Run(VIDEO_CAPTURE_STATE_STOPPED); 344 } 345 clients_.clear(); 346 state_ = VIDEO_CAPTURE_STATE_ENDED; 347 break; 348 default: 349 break; 350 } 351 } 352 353 void VideoCaptureImpl::OnDeviceSupportedFormatsEnumerated( 354 const media::VideoCaptureFormats& supported_formats) { 355 DCHECK(thread_checker_.CalledOnValidThread()); 356 for (size_t i = 0; i < device_formats_cb_queue_.size(); ++i) 357 device_formats_cb_queue_[i].Run(supported_formats); 358 device_formats_cb_queue_.clear(); 359 } 360 361 void VideoCaptureImpl::OnDeviceFormatsInUseReceived( 362 const media::VideoCaptureFormats& formats_in_use) { 363 DCHECK(thread_checker_.CalledOnValidThread()); 364 for (size_t i = 0; i < device_formats_in_use_cb_queue_.size(); ++i) 365 device_formats_in_use_cb_queue_[i].Run(formats_in_use); 366 device_formats_in_use_cb_queue_.clear(); 367 } 368 369 void VideoCaptureImpl::OnDelegateAdded(int32 device_id) { 370 DCHECK(thread_checker_.CalledOnValidThread()); 371 DVLOG(1) << "OnDelegateAdded: device_id " << device_id; 372 373 device_id_ = device_id; 374 for (ClientInfoMap::iterator it = clients_pending_on_filter_.begin(); 375 it != clients_pending_on_filter_.end(); ) { 376 int client_id = it->first; 377 VideoCaptureStateUpdateCB state_update_cb = 378 it->second.state_update_cb; 379 VideoCaptureDeliverFrameCB deliver_frame_cb = 380 it->second.deliver_frame_cb; 381 const media::VideoCaptureParams params = it->second.params; 382 clients_pending_on_filter_.erase(it++); 383 StartCapture(client_id, params, state_update_cb, 384 deliver_frame_cb); 385 } 386 } 387 388 void VideoCaptureImpl::StopDevice() { 389 DCHECK(thread_checker_.CalledOnValidThread()); 390 391 if (state_ == VIDEO_CAPTURE_STATE_STARTED) { 392 state_ = VIDEO_CAPTURE_STATE_STOPPING; 393 Send(new VideoCaptureHostMsg_Stop(device_id_)); 394 params_.requested_format.frame_size.SetSize(0, 0); 395 } 396 } 397 398 void VideoCaptureImpl::RestartCapture() { 399 DCHECK(thread_checker_.CalledOnValidThread()); 400 DCHECK_EQ(state_, VIDEO_CAPTURE_STATE_STOPPED); 401 402 int width = 0; 403 int height = 0; 404 clients_.insert(clients_pending_on_restart_.begin(), 405 clients_pending_on_restart_.end()); 406 clients_pending_on_restart_.clear(); 407 for (ClientInfoMap::iterator it = clients_.begin(); 408 it != clients_.end(); ++it) { 409 width = std::max(width, 410 it->second.params.requested_format.frame_size.width()); 411 height = std::max(height, 412 it->second.params.requested_format.frame_size.height()); 413 } 414 params_.requested_format.frame_size.SetSize(width, height); 415 DVLOG(1) << "RestartCapture, " 416 << params_.requested_format.frame_size.ToString(); 417 StartCaptureInternal(); 418 } 419 420 void VideoCaptureImpl::StartCaptureInternal() { 421 DCHECK(thread_checker_.CalledOnValidThread()); 422 DCHECK(device_id_); 423 424 Send(new VideoCaptureHostMsg_Start(device_id_, session_id_, params_)); 425 state_ = VIDEO_CAPTURE_STATE_STARTED; 426 } 427 428 void VideoCaptureImpl::Send(IPC::Message* message) { 429 DCHECK(thread_checker_.CalledOnValidThread()); 430 message_filter_->Send(message); 431 } 432 433 bool VideoCaptureImpl::RemoveClient(int client_id, ClientInfoMap* clients) { 434 DCHECK(thread_checker_.CalledOnValidThread()); 435 bool found = false; 436 437 ClientInfoMap::iterator it = clients->find(client_id); 438 if (it != clients->end()) { 439 it->second.state_update_cb.Run(VIDEO_CAPTURE_STATE_STOPPED); 440 clients->erase(it); 441 found = true; 442 } 443 return found; 444 } 445 446 } // namespace content 447