1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "content/renderer/media/webmediaplayer_ms.h" 6 7 #include <limits> 8 9 #include "base/bind.h" 10 #include "base/callback.h" 11 #include "base/message_loop/message_loop.h" 12 #include "base/metrics/histogram.h" 13 #include "cc/blink/web_layer_impl.h" 14 #include "cc/layers/video_layer.h" 15 #include "content/public/renderer/render_view.h" 16 #include "content/renderer/media/media_stream_audio_renderer.h" 17 #include "content/renderer/media/media_stream_renderer_factory.h" 18 #include "content/renderer/media/video_frame_provider.h" 19 #include "content/renderer/render_frame_impl.h" 20 #include "media/base/media_log.h" 21 #include "media/base/video_frame.h" 22 #include "media/base/video_rotation.h" 23 #include "media/base/video_util.h" 24 #include "media/blink/webmediaplayer_delegate.h" 25 #include "media/blink/webmediaplayer_util.h" 26 #include "third_party/WebKit/public/platform/WebMediaPlayerClient.h" 27 #include "third_party/WebKit/public/platform/WebRect.h" 28 #include "third_party/WebKit/public/platform/WebSize.h" 29 #include "third_party/WebKit/public/platform/WebURL.h" 30 #include "third_party/WebKit/public/web/WebFrame.h" 31 #include "third_party/WebKit/public/web/WebView.h" 32 #include "third_party/skia/include/core/SkBitmap.h" 33 34 using blink::WebCanvas; 35 using blink::WebMediaPlayer; 36 using blink::WebRect; 37 using blink::WebSize; 38 39 namespace { 40 41 // This function copies a YV12 or NATIVE_TEXTURE to a new YV12 42 // media::VideoFrame. 43 scoped_refptr<media::VideoFrame> CopyFrameToYV12( 44 const scoped_refptr<media::VideoFrame>& frame) { 45 DCHECK(frame->format() == media::VideoFrame::YV12 || 46 frame->format() == media::VideoFrame::I420 || 47 frame->format() == media::VideoFrame::NATIVE_TEXTURE); 48 scoped_refptr<media::VideoFrame> new_frame = 49 media::VideoFrame::CreateFrame(media::VideoFrame::YV12, 50 frame->coded_size(), 51 frame->visible_rect(), 52 frame->natural_size(), 53 frame->timestamp()); 54 55 if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) { 56 SkBitmap bitmap; 57 bitmap.allocN32Pixels(frame->visible_rect().width(), 58 frame->visible_rect().height()); 59 frame->ReadPixelsFromNativeTexture(bitmap); 60 61 media::CopyRGBToVideoFrame( 62 reinterpret_cast<uint8*>(bitmap.getPixels()), 63 bitmap.rowBytes(), 64 frame->visible_rect(), 65 new_frame.get()); 66 } else { 67 size_t number_of_planes = 68 media::VideoFrame::NumPlanes(frame->format()); 69 for (size_t i = 0; i < number_of_planes; ++i) { 70 media::CopyPlane(i, frame->data(i), frame->stride(i), 71 frame->rows(i), new_frame.get()); 72 } 73 } 74 return new_frame; 75 } 76 77 } // anonymous namespace 78 79 namespace content { 80 81 WebMediaPlayerMS::WebMediaPlayerMS( 82 blink::WebFrame* frame, 83 blink::WebMediaPlayerClient* client, 84 base::WeakPtr<media::WebMediaPlayerDelegate> delegate, 85 media::MediaLog* media_log, 86 scoped_ptr<MediaStreamRendererFactory> factory) 87 : frame_(frame), 88 network_state_(WebMediaPlayer::NetworkStateEmpty), 89 ready_state_(WebMediaPlayer::ReadyStateHaveNothing), 90 buffered_(static_cast<size_t>(1)), 91 volume_(1.0f), 92 client_(client), 93 delegate_(delegate), 94 paused_(true), 95 current_frame_used_(false), 96 pending_repaint_(false), 97 video_frame_provider_client_(NULL), 98 received_first_frame_(false), 99 total_frame_count_(0), 100 dropped_frame_count_(0), 101 media_log_(media_log), 102 renderer_factory_(factory.Pass()) { 103 DVLOG(1) << "WebMediaPlayerMS::ctor"; 104 media_log_->AddEvent( 105 media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_CREATED)); 106 } 107 108 WebMediaPlayerMS::~WebMediaPlayerMS() { 109 DVLOG(1) << "WebMediaPlayerMS::dtor"; 110 DCHECK(thread_checker_.CalledOnValidThread()); 111 112 SetVideoFrameProviderClient(NULL); 113 GetClient()->setWebLayer(NULL); 114 115 if (video_frame_provider_.get()) 116 video_frame_provider_->Stop(); 117 118 if (audio_renderer_.get()) 119 audio_renderer_->Stop(); 120 121 media_log_->AddEvent( 122 media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_DESTROYED)); 123 124 if (delegate_.get()) 125 delegate_->PlayerGone(this); 126 } 127 128 void WebMediaPlayerMS::load(LoadType load_type, 129 const blink::WebURL& url, 130 CORSMode cors_mode) { 131 DVLOG(1) << "WebMediaPlayerMS::load"; 132 DCHECK(thread_checker_.CalledOnValidThread()); 133 134 // TODO(acolwell): Change this to DCHECK_EQ(load_type, 135 // LoadTypeMediaStream) once Blink-side changes land. 136 DCHECK_NE(load_type, LoadTypeMediaSource); 137 138 GURL gurl(url); 139 140 SetNetworkState(WebMediaPlayer::NetworkStateLoading); 141 SetReadyState(WebMediaPlayer::ReadyStateHaveNothing); 142 media_log_->AddEvent(media_log_->CreateLoadEvent(url.spec())); 143 144 video_frame_provider_ = renderer_factory_->GetVideoFrameProvider( 145 url, 146 base::Bind(&WebMediaPlayerMS::OnSourceError, AsWeakPtr()), 147 base::Bind(&WebMediaPlayerMS::OnFrameAvailable, AsWeakPtr())); 148 149 RenderFrame* frame = RenderFrame::FromWebFrame(frame_); 150 audio_renderer_ = renderer_factory_->GetAudioRenderer( 151 url, 152 frame->GetRenderView()->GetRoutingID(), 153 frame->GetRoutingID()); 154 155 if (video_frame_provider_.get() || audio_renderer_.get()) { 156 if (audio_renderer_.get()) { 157 audio_renderer_->SetVolume(volume_); 158 audio_renderer_->Start(); 159 } 160 161 if (video_frame_provider_.get()) { 162 video_frame_provider_->Start(); 163 } else { 164 // This is audio-only mode. 165 DCHECK(audio_renderer_.get()); 166 SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata); 167 SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData); 168 } 169 } else { 170 SetNetworkState(WebMediaPlayer::NetworkStateNetworkError); 171 } 172 } 173 174 void WebMediaPlayerMS::play() { 175 DVLOG(1) << "WebMediaPlayerMS::play"; 176 DCHECK(thread_checker_.CalledOnValidThread()); 177 178 if (paused_) { 179 if (video_frame_provider_.get()) 180 video_frame_provider_->Play(); 181 182 if (audio_renderer_.get()) 183 audio_renderer_->Play(); 184 185 if (delegate_.get()) 186 delegate_->DidPlay(this); 187 } 188 189 paused_ = false; 190 191 media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PLAY)); 192 } 193 194 void WebMediaPlayerMS::pause() { 195 DVLOG(1) << "WebMediaPlayerMS::pause"; 196 DCHECK(thread_checker_.CalledOnValidThread()); 197 198 if (video_frame_provider_.get()) 199 video_frame_provider_->Pause(); 200 201 if (!paused_) { 202 if (audio_renderer_.get()) 203 audio_renderer_->Pause(); 204 205 if (delegate_.get()) 206 delegate_->DidPause(this); 207 } 208 209 paused_ = true; 210 211 media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PAUSE)); 212 213 if (!current_frame_.get()) 214 return; 215 216 // Copy the frame so that rendering can show the last received frame. 217 // The original frame must not be referenced when the player is paused since 218 // there might be a finite number of available buffers. E.g, video that 219 // originates from a video camera. 220 scoped_refptr<media::VideoFrame> new_frame = CopyFrameToYV12(current_frame_); 221 base::AutoLock auto_lock(current_frame_lock_); 222 current_frame_ = new_frame; 223 } 224 225 bool WebMediaPlayerMS::supportsSave() const { 226 DCHECK(thread_checker_.CalledOnValidThread()); 227 return false; 228 } 229 230 void WebMediaPlayerMS::seek(double seconds) { 231 DCHECK(thread_checker_.CalledOnValidThread()); 232 } 233 234 void WebMediaPlayerMS::setRate(double rate) { 235 DCHECK(thread_checker_.CalledOnValidThread()); 236 } 237 238 void WebMediaPlayerMS::setVolume(double volume) { 239 DCHECK(thread_checker_.CalledOnValidThread()); 240 DVLOG(1) << "WebMediaPlayerMS::setVolume(volume=" << volume << ")"; 241 volume_ = volume; 242 if (audio_renderer_.get()) 243 audio_renderer_->SetVolume(volume_); 244 } 245 246 void WebMediaPlayerMS::setPreload(WebMediaPlayer::Preload preload) { 247 DCHECK(thread_checker_.CalledOnValidThread()); 248 } 249 250 bool WebMediaPlayerMS::hasVideo() const { 251 DCHECK(thread_checker_.CalledOnValidThread()); 252 return (video_frame_provider_.get() != NULL); 253 } 254 255 bool WebMediaPlayerMS::hasAudio() const { 256 DCHECK(thread_checker_.CalledOnValidThread()); 257 return (audio_renderer_.get() != NULL); 258 } 259 260 blink::WebSize WebMediaPlayerMS::naturalSize() const { 261 DCHECK(thread_checker_.CalledOnValidThread()); 262 263 gfx::Size size; 264 if (current_frame_.get()) 265 size = current_frame_->natural_size(); 266 DVLOG(3) << "WebMediaPlayerMS::naturalSize, " << size.ToString(); 267 return blink::WebSize(size); 268 } 269 270 bool WebMediaPlayerMS::paused() const { 271 DCHECK(thread_checker_.CalledOnValidThread()); 272 return paused_; 273 } 274 275 bool WebMediaPlayerMS::seeking() const { 276 DCHECK(thread_checker_.CalledOnValidThread()); 277 return false; 278 } 279 280 double WebMediaPlayerMS::duration() const { 281 DCHECK(thread_checker_.CalledOnValidThread()); 282 return std::numeric_limits<double>::infinity(); 283 } 284 285 double WebMediaPlayerMS::currentTime() const { 286 DCHECK(thread_checker_.CalledOnValidThread()); 287 if (current_time_.ToInternalValue() != 0) { 288 return current_time_.InSecondsF(); 289 } else if (audio_renderer_.get()) { 290 return audio_renderer_->GetCurrentRenderTime().InSecondsF(); 291 } 292 return 0.0; 293 } 294 295 WebMediaPlayer::NetworkState WebMediaPlayerMS::networkState() const { 296 DCHECK(thread_checker_.CalledOnValidThread()); 297 DVLOG(1) << "WebMediaPlayerMS::networkState, state:" << network_state_; 298 return network_state_; 299 } 300 301 WebMediaPlayer::ReadyState WebMediaPlayerMS::readyState() const { 302 DCHECK(thread_checker_.CalledOnValidThread()); 303 DVLOG(1) << "WebMediaPlayerMS::readyState, state:" << ready_state_; 304 return ready_state_; 305 } 306 307 blink::WebTimeRanges WebMediaPlayerMS::buffered() const { 308 DCHECK(thread_checker_.CalledOnValidThread()); 309 return buffered_; 310 } 311 312 double WebMediaPlayerMS::maxTimeSeekable() const { 313 DCHECK(thread_checker_.CalledOnValidThread()); 314 return 0.0; 315 } 316 317 bool WebMediaPlayerMS::didLoadingProgress() { 318 DCHECK(thread_checker_.CalledOnValidThread()); 319 return true; 320 } 321 322 void WebMediaPlayerMS::paint(blink::WebCanvas* canvas, 323 const blink::WebRect& rect, 324 unsigned char alpha) { 325 paint(canvas, rect, alpha, SkXfermode::kSrcOver_Mode); 326 } 327 328 void WebMediaPlayerMS::paint(blink::WebCanvas* canvas, 329 const blink::WebRect& rect, 330 unsigned char alpha, 331 SkXfermode::Mode mode) { 332 DVLOG(3) << "WebMediaPlayerMS::paint"; 333 DCHECK(thread_checker_.CalledOnValidThread()); 334 335 gfx::RectF dest_rect(rect.x, rect.y, rect.width, rect.height); 336 video_renderer_.Paint( 337 current_frame_, canvas, dest_rect, alpha, mode, media::VIDEO_ROTATION_0); 338 339 { 340 base::AutoLock auto_lock(current_frame_lock_); 341 if (current_frame_.get()) 342 current_frame_used_ = true; 343 } 344 } 345 346 bool WebMediaPlayerMS::hasSingleSecurityOrigin() const { 347 DCHECK(thread_checker_.CalledOnValidThread()); 348 return true; 349 } 350 351 bool WebMediaPlayerMS::didPassCORSAccessCheck() const { 352 DCHECK(thread_checker_.CalledOnValidThread()); 353 return true; 354 } 355 356 double WebMediaPlayerMS::mediaTimeForTimeValue(double timeValue) const { 357 return media::ConvertSecondsToTimestamp(timeValue).InSecondsF(); 358 } 359 360 unsigned WebMediaPlayerMS::decodedFrameCount() const { 361 DCHECK(thread_checker_.CalledOnValidThread()); 362 DVLOG(1) << "WebMediaPlayerMS::decodedFrameCount, " << total_frame_count_; 363 return total_frame_count_; 364 } 365 366 unsigned WebMediaPlayerMS::droppedFrameCount() const { 367 DCHECK(thread_checker_.CalledOnValidThread()); 368 DVLOG(1) << "WebMediaPlayerMS::droppedFrameCount, " << dropped_frame_count_; 369 return dropped_frame_count_; 370 } 371 372 unsigned WebMediaPlayerMS::audioDecodedByteCount() const { 373 DCHECK(thread_checker_.CalledOnValidThread()); 374 NOTIMPLEMENTED(); 375 return 0; 376 } 377 378 unsigned WebMediaPlayerMS::videoDecodedByteCount() const { 379 DCHECK(thread_checker_.CalledOnValidThread()); 380 NOTIMPLEMENTED(); 381 return 0; 382 } 383 384 void WebMediaPlayerMS::SetVideoFrameProviderClient( 385 cc::VideoFrameProvider::Client* client) { 386 // This is called from both the main renderer thread and the compositor 387 // thread (when the main thread is blocked). 388 if (video_frame_provider_client_) 389 video_frame_provider_client_->StopUsingProvider(); 390 video_frame_provider_client_ = client; 391 } 392 393 scoped_refptr<media::VideoFrame> WebMediaPlayerMS::GetCurrentFrame() { 394 DVLOG(3) << "WebMediaPlayerMS::GetCurrentFrame"; 395 base::AutoLock auto_lock(current_frame_lock_); 396 DCHECK(!pending_repaint_); 397 if (!current_frame_.get()) 398 return NULL; 399 pending_repaint_ = true; 400 current_frame_used_ = true; 401 return current_frame_; 402 } 403 404 void WebMediaPlayerMS::PutCurrentFrame( 405 const scoped_refptr<media::VideoFrame>& frame) { 406 DVLOG(3) << "WebMediaPlayerMS::PutCurrentFrame"; 407 DCHECK(pending_repaint_); 408 pending_repaint_ = false; 409 } 410 411 void WebMediaPlayerMS::OnFrameAvailable( 412 const scoped_refptr<media::VideoFrame>& frame) { 413 DVLOG(3) << "WebMediaPlayerMS::OnFrameAvailable"; 414 DCHECK(thread_checker_.CalledOnValidThread()); 415 ++total_frame_count_; 416 if (!received_first_frame_) { 417 received_first_frame_ = true; 418 { 419 base::AutoLock auto_lock(current_frame_lock_); 420 DCHECK(!current_frame_used_); 421 current_frame_ = frame; 422 } 423 SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata); 424 SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData); 425 GetClient()->sizeChanged(); 426 427 if (video_frame_provider_.get()) { 428 video_weblayer_.reset(new cc_blink::WebLayerImpl( 429 cc::VideoLayer::Create(this, media::VIDEO_ROTATION_0))); 430 video_weblayer_->setOpaque(true); 431 GetClient()->setWebLayer(video_weblayer_.get()); 432 } 433 } 434 435 // Do not update |current_frame_| when paused. 436 if (paused_) 437 return; 438 439 bool size_changed = !current_frame_.get() || 440 current_frame_->natural_size() != frame->natural_size(); 441 442 { 443 base::AutoLock auto_lock(current_frame_lock_); 444 if (!current_frame_used_ && current_frame_.get()) 445 ++dropped_frame_count_; 446 current_frame_ = frame; 447 current_time_ = frame->timestamp(); 448 current_frame_used_ = false; 449 } 450 451 if (size_changed) 452 GetClient()->sizeChanged(); 453 454 GetClient()->repaint(); 455 } 456 457 void WebMediaPlayerMS::RepaintInternal() { 458 DVLOG(1) << "WebMediaPlayerMS::RepaintInternal"; 459 DCHECK(thread_checker_.CalledOnValidThread()); 460 GetClient()->repaint(); 461 } 462 463 void WebMediaPlayerMS::OnSourceError() { 464 DVLOG(1) << "WebMediaPlayerMS::OnSourceError"; 465 DCHECK(thread_checker_.CalledOnValidThread()); 466 SetNetworkState(WebMediaPlayer::NetworkStateFormatError); 467 RepaintInternal(); 468 } 469 470 void WebMediaPlayerMS::SetNetworkState(WebMediaPlayer::NetworkState state) { 471 DCHECK(thread_checker_.CalledOnValidThread()); 472 network_state_ = state; 473 // Always notify to ensure client has the latest value. 474 GetClient()->networkStateChanged(); 475 } 476 477 void WebMediaPlayerMS::SetReadyState(WebMediaPlayer::ReadyState state) { 478 DCHECK(thread_checker_.CalledOnValidThread()); 479 ready_state_ = state; 480 // Always notify to ensure client has the latest value. 481 GetClient()->readyStateChanged(); 482 } 483 484 blink::WebMediaPlayerClient* WebMediaPlayerMS::GetClient() { 485 DCHECK(thread_checker_.CalledOnValidThread()); 486 DCHECK(client_); 487 return client_; 488 } 489 490 } // namespace content 491