1 // Copyright 2014 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "content/browser/media/capture/content_video_capture_device_core.h" 6 7 #include "base/basictypes.h" 8 #include "base/bind.h" 9 #include "base/callback_forward.h" 10 #include "base/callback_helpers.h" 11 #include "base/debug/trace_event.h" 12 #include "base/logging.h" 13 #include "base/memory/scoped_ptr.h" 14 #include "base/memory/weak_ptr.h" 15 #include "base/message_loop/message_loop_proxy.h" 16 #include "base/metrics/histogram.h" 17 #include "base/sequenced_task_runner.h" 18 #include "base/strings/string_number_conversions.h" 19 #include "base/strings/stringprintf.h" 20 #include "base/synchronization/lock.h" 21 #include "base/threading/thread.h" 22 #include "base/threading/thread_checker.h" 23 #include "base/time/time.h" 24 #include "content/public/browser/browser_thread.h" 25 #include "media/base/bind_to_current_loop.h" 26 #include "media/base/video_frame.h" 27 #include "media/base/video_util.h" 28 #include "media/video/capture/video_capture_types.h" 29 #include "ui/gfx/rect.h" 30 31 namespace content { 32 33 namespace { 34 35 void DeleteCaptureMachineOnUIThread( 36 scoped_ptr<VideoCaptureMachine> capture_machine) { 37 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); 38 39 capture_machine.reset(); 40 } 41 42 } // namespace 43 44 ThreadSafeCaptureOracle::ThreadSafeCaptureOracle( 45 scoped_ptr<media::VideoCaptureDevice::Client> client, 46 scoped_ptr<VideoCaptureOracle> oracle, 47 const media::VideoCaptureParams& params) 48 : client_(client.Pass()), 49 oracle_(oracle.Pass()), 50 params_(params), 51 capture_size_updated_(false) { 52 switch (params_.requested_format.pixel_format) { 53 case media::PIXEL_FORMAT_I420: 54 video_frame_format_ = media::VideoFrame::I420; 55 break; 56 case media::PIXEL_FORMAT_TEXTURE: 57 video_frame_format_ = media::VideoFrame::NATIVE_TEXTURE; 58 break; 59 default: 60 LOG(FATAL) << "Unexpected pixel_format " 61 << params_.requested_format.pixel_format; 62 } 63 } 64 65 ThreadSafeCaptureOracle::~ThreadSafeCaptureOracle() {} 66 67 bool ThreadSafeCaptureOracle::ObserveEventAndDecideCapture( 68 VideoCaptureOracle::Event event, 69 const gfx::Rect& damage_rect, 70 base::TimeTicks event_time, 71 scoped_refptr<media::VideoFrame>* storage, 72 CaptureFrameCallback* callback) { 73 base::AutoLock guard(lock_); 74 75 if (!client_) 76 return false; // Capture is stopped. 77 78 // Always round up the coded size to multiple of 16 pixels. 79 // See http://crbug.com/402151. 80 const gfx::Size visible_size = params_.requested_format.frame_size; 81 const gfx::Size coded_size((visible_size.width() + 15) & ~15, 82 (visible_size.height() + 15) & ~15); 83 84 scoped_refptr<media::VideoCaptureDevice::Client::Buffer> output_buffer = 85 client_->ReserveOutputBuffer(video_frame_format_, coded_size); 86 const bool should_capture = 87 oracle_->ObserveEventAndDecideCapture(event, damage_rect, event_time); 88 const bool content_is_dirty = 89 (event == VideoCaptureOracle::kCompositorUpdate || 90 event == VideoCaptureOracle::kSoftwarePaint); 91 const char* event_name = 92 (event == VideoCaptureOracle::kTimerPoll ? "poll" : 93 (event == VideoCaptureOracle::kCompositorUpdate ? "gpu" : 94 "paint")); 95 96 // Consider the various reasons not to initiate a capture. 97 if (should_capture && !output_buffer.get()) { 98 TRACE_EVENT_INSTANT1("mirroring", 99 "PipelineLimited", 100 TRACE_EVENT_SCOPE_THREAD, 101 "trigger", 102 event_name); 103 return false; 104 } else if (!should_capture && output_buffer.get()) { 105 if (content_is_dirty) { 106 // This is a normal and acceptable way to drop a frame. We've hit our 107 // capture rate limit: for example, the content is animating at 60fps but 108 // we're capturing at 30fps. 109 TRACE_EVENT_INSTANT1("mirroring", "FpsRateLimited", 110 TRACE_EVENT_SCOPE_THREAD, 111 "trigger", event_name); 112 } 113 return false; 114 } else if (!should_capture && !output_buffer.get()) { 115 // We decided not to capture, but we wouldn't have been able to if we wanted 116 // to because no output buffer was available. 117 TRACE_EVENT_INSTANT1("mirroring", "NearlyPipelineLimited", 118 TRACE_EVENT_SCOPE_THREAD, 119 "trigger", event_name); 120 return false; 121 } 122 int frame_number = oracle_->RecordCapture(); 123 TRACE_EVENT_ASYNC_BEGIN2("mirroring", "Capture", output_buffer.get(), 124 "frame_number", frame_number, 125 "trigger", event_name); 126 // NATIVE_TEXTURE frames wrap a texture mailbox, which we don't have at the 127 // moment. We do not construct those frames. 128 if (video_frame_format_ != media::VideoFrame::NATIVE_TEXTURE) { 129 *storage = media::VideoFrame::WrapExternalPackedMemory( 130 video_frame_format_, 131 coded_size, 132 gfx::Rect(visible_size), 133 visible_size, 134 static_cast<uint8*>(output_buffer->data()), 135 output_buffer->size(), 136 base::SharedMemory::NULLHandle(), 137 base::TimeDelta(), 138 base::Closure()); 139 } 140 *callback = base::Bind(&ThreadSafeCaptureOracle::DidCaptureFrame, 141 this, 142 frame_number, 143 output_buffer); 144 return true; 145 } 146 147 gfx::Size ThreadSafeCaptureOracle::GetCaptureSize() const { 148 base::AutoLock guard(lock_); 149 return params_.requested_format.frame_size; 150 } 151 152 void ThreadSafeCaptureOracle::UpdateCaptureSize(const gfx::Size& source_size) { 153 base::AutoLock guard(lock_); 154 155 // If this is the first call to UpdateCaptureSize(), or the receiver supports 156 // variable resolution, then determine the capture size by treating the 157 // requested width and height as maxima. 158 if (!capture_size_updated_ || params_.resolution_change_policy == 159 media::RESOLUTION_POLICY_DYNAMIC_WITHIN_LIMIT) { 160 // The capture resolution should not exceed the source frame size. 161 // In other words it should downscale the image but not upscale it. 162 if (source_size.width() > params_.requested_format.frame_size.width() || 163 source_size.height() > params_.requested_format.frame_size.height()) { 164 gfx::Rect capture_rect = media::ComputeLetterboxRegion( 165 gfx::Rect(params_.requested_format.frame_size), source_size); 166 params_.requested_format.frame_size.SetSize( 167 MakeEven(capture_rect.width()), MakeEven(capture_rect.height())); 168 } else { 169 params_.requested_format.frame_size.SetSize( 170 MakeEven(source_size.width()), MakeEven(source_size.height())); 171 } 172 capture_size_updated_ = true; 173 } 174 } 175 176 void ThreadSafeCaptureOracle::Stop() { 177 base::AutoLock guard(lock_); 178 client_.reset(); 179 } 180 181 void ThreadSafeCaptureOracle::ReportError(const std::string& reason) { 182 base::AutoLock guard(lock_); 183 if (client_) 184 client_->OnError(reason); 185 } 186 187 void ThreadSafeCaptureOracle::DidCaptureFrame( 188 int frame_number, 189 const scoped_refptr<media::VideoCaptureDevice::Client::Buffer>& buffer, 190 const scoped_refptr<media::VideoFrame>& frame, 191 base::TimeTicks timestamp, 192 bool success) { 193 base::AutoLock guard(lock_); 194 TRACE_EVENT_ASYNC_END2("mirroring", "Capture", buffer.get(), 195 "success", success, 196 "timestamp", timestamp.ToInternalValue()); 197 198 if (!client_) 199 return; // Capture is stopped. 200 201 if (success) { 202 if (oracle_->CompleteCapture(frame_number, ×tamp)) { 203 media::VideoCaptureFormat format = params_.requested_format; 204 format.frame_size = frame->coded_size(); 205 client_->OnIncomingCapturedVideoFrame(buffer, format, frame, timestamp); 206 } 207 } 208 } 209 210 void ContentVideoCaptureDeviceCore::AllocateAndStart( 211 const media::VideoCaptureParams& params, 212 scoped_ptr<media::VideoCaptureDevice::Client> client) { 213 DCHECK(thread_checker_.CalledOnValidThread()); 214 215 if (state_ != kIdle) { 216 DVLOG(1) << "Allocate() invoked when not in state Idle."; 217 return; 218 } 219 220 if (params.requested_format.frame_rate <= 0) { 221 std::string error_msg("Invalid frame_rate: "); 222 error_msg += base::DoubleToString(params.requested_format.frame_rate); 223 DVLOG(1) << error_msg; 224 client->OnError(error_msg); 225 return; 226 } 227 228 if (params.requested_format.pixel_format != media::PIXEL_FORMAT_I420 && 229 params.requested_format.pixel_format != media::PIXEL_FORMAT_TEXTURE) { 230 std::string error_msg = base::StringPrintf( 231 "unsupported format: %d", params.requested_format.pixel_format); 232 DVLOG(1) << error_msg; 233 client->OnError(error_msg); 234 return; 235 } 236 237 if (params.requested_format.frame_size.width() < kMinFrameWidth || 238 params.requested_format.frame_size.height() < kMinFrameHeight) { 239 std::string error_msg = 240 "invalid frame size: " + params.requested_format.frame_size.ToString(); 241 DVLOG(1) << error_msg; 242 client->OnError(error_msg); 243 return; 244 } 245 246 media::VideoCaptureParams new_params = params; 247 // Frame dimensions must each be an even integer since the client wants (or 248 // will convert to) YUV420. 249 new_params.requested_format.frame_size.SetSize( 250 MakeEven(params.requested_format.frame_size.width()), 251 MakeEven(params.requested_format.frame_size.height())); 252 253 base::TimeDelta capture_period = base::TimeDelta::FromMicroseconds( 254 1000000.0 / params.requested_format.frame_rate + 0.5); 255 256 scoped_ptr<VideoCaptureOracle> oracle( 257 new VideoCaptureOracle(capture_period, 258 kAcceleratedSubscriberIsSupported)); 259 oracle_proxy_ = 260 new ThreadSafeCaptureOracle(client.Pass(), oracle.Pass(), new_params); 261 262 // Starts the capture machine asynchronously. 263 BrowserThread::PostTaskAndReplyWithResult( 264 BrowserThread::UI, 265 FROM_HERE, 266 base::Bind(&VideoCaptureMachine::Start, 267 base::Unretained(capture_machine_.get()), 268 oracle_proxy_, 269 new_params), 270 base::Bind(&ContentVideoCaptureDeviceCore::CaptureStarted, AsWeakPtr())); 271 272 TransitionStateTo(kCapturing); 273 } 274 275 void ContentVideoCaptureDeviceCore::StopAndDeAllocate() { 276 DCHECK(thread_checker_.CalledOnValidThread()); 277 278 if (state_ != kCapturing) 279 return; 280 281 oracle_proxy_->Stop(); 282 oracle_proxy_ = NULL; 283 284 TransitionStateTo(kIdle); 285 286 // Stops the capture machine asynchronously. 287 BrowserThread::PostTask( 288 BrowserThread::UI, FROM_HERE, base::Bind( 289 &VideoCaptureMachine::Stop, 290 base::Unretained(capture_machine_.get()), 291 base::Bind(&base::DoNothing))); 292 } 293 294 void ContentVideoCaptureDeviceCore::CaptureStarted(bool success) { 295 DCHECK(thread_checker_.CalledOnValidThread()); 296 if (!success) { 297 std::string reason("Failed to start capture machine."); 298 DVLOG(1) << reason; 299 Error(reason); 300 } 301 } 302 303 ContentVideoCaptureDeviceCore::ContentVideoCaptureDeviceCore( 304 scoped_ptr<VideoCaptureMachine> capture_machine) 305 : state_(kIdle), 306 capture_machine_(capture_machine.Pass()) { 307 DCHECK(capture_machine_.get()); 308 } 309 310 ContentVideoCaptureDeviceCore::~ContentVideoCaptureDeviceCore() { 311 DCHECK(thread_checker_.CalledOnValidThread()); 312 DCHECK_NE(state_, kCapturing); 313 // If capture_machine is not NULL, then we need to return to the UI thread to 314 // safely stop the capture machine. 315 if (capture_machine_) { 316 VideoCaptureMachine* capture_machine_ptr = capture_machine_.get(); 317 BrowserThread::PostTask( 318 BrowserThread::UI, FROM_HERE, 319 base::Bind(&VideoCaptureMachine::Stop, 320 base::Unretained(capture_machine_ptr), 321 base::Bind(&DeleteCaptureMachineOnUIThread, 322 base::Passed(&capture_machine_)))); 323 } 324 DVLOG(1) << "ContentVideoCaptureDeviceCore@" << this << " destroying."; 325 } 326 327 void ContentVideoCaptureDeviceCore::TransitionStateTo(State next_state) { 328 DCHECK(thread_checker_.CalledOnValidThread()); 329 330 #ifndef NDEBUG 331 static const char* kStateNames[] = { 332 "Idle", "Allocated", "Capturing", "Error" 333 }; 334 DVLOG(1) << "State change: " << kStateNames[state_] 335 << " --> " << kStateNames[next_state]; 336 #endif 337 338 state_ = next_state; 339 } 340 341 void ContentVideoCaptureDeviceCore::Error(const std::string& reason) { 342 DCHECK(thread_checker_.CalledOnValidThread()); 343 344 if (state_ == kIdle) 345 return; 346 347 if (oracle_proxy_.get()) 348 oracle_proxy_->ReportError(reason); 349 350 StopAndDeAllocate(); 351 TransitionStateTo(kError); 352 } 353 354 } // namespace content 355