Home | History | Annotate | Download | only in media 
      1 // Copyright 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "content/renderer/media/webmediaplayer_ms.h"
      6 
      7 #include <limits>
      8 
      9 #include "base/bind.h"
     10 #include "base/callback.h"
     11 #include "base/message_loop/message_loop.h"
     12 #include "base/metrics/histogram.h"
     13 #include "cc/blink/web_layer_impl.h"
     14 #include "cc/layers/video_layer.h"
     15 #include "content/public/renderer/render_view.h"
     16 #include "content/renderer/media/media_stream_audio_renderer.h"
     17 #include "content/renderer/media/media_stream_renderer_factory.h"
     18 #include "content/renderer/media/video_frame_provider.h"
     19 #include "content/renderer/render_frame_impl.h"
     20 #include "media/base/media_log.h"
     21 #include "media/base/video_frame.h"
     22 #include "media/base/video_rotation.h"
     23 #include "media/base/video_util.h"
     24 #include "media/blink/webmediaplayer_delegate.h"
     25 #include "media/blink/webmediaplayer_util.h"
     26 #include "third_party/WebKit/public/platform/WebMediaPlayerClient.h"
     27 #include "third_party/WebKit/public/platform/WebRect.h"
     28 #include "third_party/WebKit/public/platform/WebSize.h"
     29 #include "third_party/WebKit/public/platform/WebURL.h"
     30 #include "third_party/WebKit/public/web/WebFrame.h"
     31 #include "third_party/WebKit/public/web/WebView.h"
     32 #include "third_party/skia/include/core/SkBitmap.h"
     33 
     34 using blink::WebCanvas;
     35 using blink::WebMediaPlayer;
     36 using blink::WebRect;
     37 using blink::WebSize;
     38 
     39 namespace {
     40 
     41 // This function copies a YV12 or NATIVE_TEXTURE to a new YV12
     42 // media::VideoFrame.
     43 scoped_refptr<media::VideoFrame> CopyFrameToYV12(
     44     const scoped_refptr<media::VideoFrame>& frame) {
     45   DCHECK(frame->format() == media::VideoFrame::YV12 ||
     46          frame->format() == media::VideoFrame::I420 ||
     47          frame->format() == media::VideoFrame::NATIVE_TEXTURE);
     48   scoped_refptr<media::VideoFrame> new_frame =
     49       media::VideoFrame::CreateFrame(media::VideoFrame::YV12,
     50                                      frame->coded_size(),
     51                                      frame->visible_rect(),
     52                                      frame->natural_size(),
     53                                      frame->timestamp());
     54 
     55   if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
     56     SkBitmap bitmap;
     57     bitmap.allocN32Pixels(frame->visible_rect().width(),
     58                           frame->visible_rect().height());
     59     frame->ReadPixelsFromNativeTexture(bitmap);
     60 
     61     media::CopyRGBToVideoFrame(
     62         reinterpret_cast<uint8*>(bitmap.getPixels()),
     63         bitmap.rowBytes(),
     64         frame->visible_rect(),
     65         new_frame.get());
     66   } else {
     67     size_t number_of_planes =
     68         media::VideoFrame::NumPlanes(frame->format());
     69     for (size_t i = 0; i < number_of_planes; ++i) {
     70       media::CopyPlane(i, frame->data(i), frame->stride(i),
     71                        frame->rows(i), new_frame.get());
     72     }
     73   }
     74   return new_frame;
     75 }
     76 
     77 }  // anonymous namespace
     78 
     79 namespace content {
     80 
     81 WebMediaPlayerMS::WebMediaPlayerMS(
     82     blink::WebFrame* frame,
     83     blink::WebMediaPlayerClient* client,
     84     base::WeakPtr<media::WebMediaPlayerDelegate> delegate,
     85     media::MediaLog* media_log,
     86     scoped_ptr<MediaStreamRendererFactory> factory)
     87     : frame_(frame),
     88       network_state_(WebMediaPlayer::NetworkStateEmpty),
     89       ready_state_(WebMediaPlayer::ReadyStateHaveNothing),
     90       buffered_(static_cast<size_t>(1)),
     91       volume_(1.0f),
     92       client_(client),
     93       delegate_(delegate),
     94       paused_(true),
     95       current_frame_used_(false),
     96       pending_repaint_(false),
     97       video_frame_provider_client_(NULL),
     98       received_first_frame_(false),
     99       total_frame_count_(0),
    100       dropped_frame_count_(0),
    101       media_log_(media_log),
    102       renderer_factory_(factory.Pass()) {
    103   DVLOG(1) << "WebMediaPlayerMS::ctor";
    104   media_log_->AddEvent(
    105       media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_CREATED));
    106 }
    107 
    108 WebMediaPlayerMS::~WebMediaPlayerMS() {
    109   DVLOG(1) << "WebMediaPlayerMS::dtor";
    110   DCHECK(thread_checker_.CalledOnValidThread());
    111 
    112   SetVideoFrameProviderClient(NULL);
    113   GetClient()->setWebLayer(NULL);
    114 
    115   if (video_frame_provider_.get())
    116     video_frame_provider_->Stop();
    117 
    118   if (audio_renderer_.get())
    119     audio_renderer_->Stop();
    120 
    121   media_log_->AddEvent(
    122       media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_DESTROYED));
    123 
    124   if (delegate_.get())
    125     delegate_->PlayerGone(this);
    126 }
    127 
    128 void WebMediaPlayerMS::load(LoadType load_type,
    129                             const blink::WebURL& url,
    130                             CORSMode cors_mode) {
    131   DVLOG(1) << "WebMediaPlayerMS::load";
    132   DCHECK(thread_checker_.CalledOnValidThread());
    133 
    134   // TODO(acolwell): Change this to DCHECK_EQ(load_type,
    135   // LoadTypeMediaStream) once Blink-side changes land.
    136   DCHECK_NE(load_type, LoadTypeMediaSource);
    137 
    138   GURL gurl(url);
    139 
    140   SetNetworkState(WebMediaPlayer::NetworkStateLoading);
    141   SetReadyState(WebMediaPlayer::ReadyStateHaveNothing);
    142   media_log_->AddEvent(media_log_->CreateLoadEvent(url.spec()));
    143 
    144   video_frame_provider_ = renderer_factory_->GetVideoFrameProvider(
    145       url,
    146       base::Bind(&WebMediaPlayerMS::OnSourceError, AsWeakPtr()),
    147       base::Bind(&WebMediaPlayerMS::OnFrameAvailable, AsWeakPtr()));
    148 
    149   RenderFrame* frame = RenderFrame::FromWebFrame(frame_);
    150   audio_renderer_ = renderer_factory_->GetAudioRenderer(
    151     url,
    152     frame->GetRenderView()->GetRoutingID(),
    153     frame->GetRoutingID());
    154 
    155   if (video_frame_provider_.get() || audio_renderer_.get()) {
    156     if (audio_renderer_.get()) {
    157       audio_renderer_->SetVolume(volume_);
    158       audio_renderer_->Start();
    159     }
    160 
    161     if (video_frame_provider_.get()) {
    162       video_frame_provider_->Start();
    163     } else {
    164       // This is audio-only mode.
    165       DCHECK(audio_renderer_.get());
    166       SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
    167       SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
    168     }
    169   } else {
    170     SetNetworkState(WebMediaPlayer::NetworkStateNetworkError);
    171   }
    172 }
    173 
    174 void WebMediaPlayerMS::play() {
    175   DVLOG(1) << "WebMediaPlayerMS::play";
    176   DCHECK(thread_checker_.CalledOnValidThread());
    177 
    178   if (paused_) {
    179     if (video_frame_provider_.get())
    180       video_frame_provider_->Play();
    181 
    182     if (audio_renderer_.get())
    183       audio_renderer_->Play();
    184 
    185     if (delegate_.get())
    186       delegate_->DidPlay(this);
    187   }
    188 
    189   paused_ = false;
    190 
    191   media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PLAY));
    192 }
    193 
    194 void WebMediaPlayerMS::pause() {
    195   DVLOG(1) << "WebMediaPlayerMS::pause";
    196   DCHECK(thread_checker_.CalledOnValidThread());
    197 
    198   if (video_frame_provider_.get())
    199     video_frame_provider_->Pause();
    200 
    201   if (!paused_) {
    202     if (audio_renderer_.get())
    203       audio_renderer_->Pause();
    204 
    205     if (delegate_.get())
    206       delegate_->DidPause(this);
    207   }
    208 
    209   paused_ = true;
    210 
    211   media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PAUSE));
    212 
    213   if (!current_frame_.get())
    214     return;
    215 
    216   // Copy the frame so that rendering can show the last received frame.
    217   // The original frame must not be referenced when the player is paused since
    218   // there might be a finite number of available buffers. E.g, video that
    219   // originates from a video camera.
    220   scoped_refptr<media::VideoFrame> new_frame = CopyFrameToYV12(current_frame_);
    221   base::AutoLock auto_lock(current_frame_lock_);
    222   current_frame_ = new_frame;
    223 }
    224 
    225 bool WebMediaPlayerMS::supportsSave() const {
    226   DCHECK(thread_checker_.CalledOnValidThread());
    227   return false;
    228 }
    229 
    230 void WebMediaPlayerMS::seek(double seconds) {
    231   DCHECK(thread_checker_.CalledOnValidThread());
    232 }
    233 
    234 void WebMediaPlayerMS::setRate(double rate) {
    235   DCHECK(thread_checker_.CalledOnValidThread());
    236 }
    237 
    238 void WebMediaPlayerMS::setVolume(double volume) {
    239   DCHECK(thread_checker_.CalledOnValidThread());
    240   DVLOG(1) << "WebMediaPlayerMS::setVolume(volume=" << volume << ")";
    241   volume_ = volume;
    242   if (audio_renderer_.get())
    243     audio_renderer_->SetVolume(volume_);
    244 }
    245 
    246 void WebMediaPlayerMS::setPreload(WebMediaPlayer::Preload preload) {
    247   DCHECK(thread_checker_.CalledOnValidThread());
    248 }
    249 
    250 bool WebMediaPlayerMS::hasVideo() const {
    251   DCHECK(thread_checker_.CalledOnValidThread());
    252   return (video_frame_provider_.get() != NULL);
    253 }
    254 
    255 bool WebMediaPlayerMS::hasAudio() const {
    256   DCHECK(thread_checker_.CalledOnValidThread());
    257   return (audio_renderer_.get() != NULL);
    258 }
    259 
    260 blink::WebSize WebMediaPlayerMS::naturalSize() const {
    261   DCHECK(thread_checker_.CalledOnValidThread());
    262 
    263   gfx::Size size;
    264   if (current_frame_.get())
    265     size = current_frame_->natural_size();
    266   DVLOG(3) << "WebMediaPlayerMS::naturalSize, " << size.ToString();
    267   return blink::WebSize(size);
    268 }
    269 
    270 bool WebMediaPlayerMS::paused() const {
    271   DCHECK(thread_checker_.CalledOnValidThread());
    272   return paused_;
    273 }
    274 
    275 bool WebMediaPlayerMS::seeking() const {
    276   DCHECK(thread_checker_.CalledOnValidThread());
    277   return false;
    278 }
    279 
    280 double WebMediaPlayerMS::duration() const {
    281   DCHECK(thread_checker_.CalledOnValidThread());
    282   return std::numeric_limits<double>::infinity();
    283 }
    284 
    285 double WebMediaPlayerMS::currentTime() const {
    286   DCHECK(thread_checker_.CalledOnValidThread());
    287   if (current_time_.ToInternalValue() != 0) {
    288     return current_time_.InSecondsF();
    289   } else if (audio_renderer_.get()) {
    290     return audio_renderer_->GetCurrentRenderTime().InSecondsF();
    291   }
    292   return 0.0;
    293 }
    294 
    295 WebMediaPlayer::NetworkState WebMediaPlayerMS::networkState() const {
    296   DCHECK(thread_checker_.CalledOnValidThread());
    297   DVLOG(1) << "WebMediaPlayerMS::networkState, state:" << network_state_;
    298   return network_state_;
    299 }
    300 
    301 WebMediaPlayer::ReadyState WebMediaPlayerMS::readyState() const {
    302   DCHECK(thread_checker_.CalledOnValidThread());
    303   DVLOG(1) << "WebMediaPlayerMS::readyState, state:" << ready_state_;
    304   return ready_state_;
    305 }
    306 
    307 blink::WebTimeRanges WebMediaPlayerMS::buffered() const {
    308   DCHECK(thread_checker_.CalledOnValidThread());
    309   return buffered_;
    310 }
    311 
    312 double WebMediaPlayerMS::maxTimeSeekable() const {
    313   DCHECK(thread_checker_.CalledOnValidThread());
    314   return 0.0;
    315 }
    316 
    317 bool WebMediaPlayerMS::didLoadingProgress() {
    318   DCHECK(thread_checker_.CalledOnValidThread());
    319   return true;
    320 }
    321 
    322 void WebMediaPlayerMS::paint(blink::WebCanvas* canvas,
    323                              const blink::WebRect& rect,
    324                              unsigned char alpha) {
    325   paint(canvas, rect, alpha, SkXfermode::kSrcOver_Mode);
    326 }
    327 
    328 void WebMediaPlayerMS::paint(blink::WebCanvas* canvas,
    329                              const blink::WebRect& rect,
    330                              unsigned char alpha,
    331                              SkXfermode::Mode mode) {
    332   DVLOG(3) << "WebMediaPlayerMS::paint";
    333   DCHECK(thread_checker_.CalledOnValidThread());
    334 
    335   gfx::RectF dest_rect(rect.x, rect.y, rect.width, rect.height);
    336   video_renderer_.Paint(
    337       current_frame_, canvas, dest_rect, alpha, mode, media::VIDEO_ROTATION_0);
    338 
    339   {
    340     base::AutoLock auto_lock(current_frame_lock_);
    341     if (current_frame_.get())
    342       current_frame_used_ = true;
    343   }
    344 }
    345 
    346 bool WebMediaPlayerMS::hasSingleSecurityOrigin() const {
    347   DCHECK(thread_checker_.CalledOnValidThread());
    348   return true;
    349 }
    350 
    351 bool WebMediaPlayerMS::didPassCORSAccessCheck() const {
    352   DCHECK(thread_checker_.CalledOnValidThread());
    353   return true;
    354 }
    355 
    356 double WebMediaPlayerMS::mediaTimeForTimeValue(double timeValue) const {
    357   return media::ConvertSecondsToTimestamp(timeValue).InSecondsF();
    358 }
    359 
    360 unsigned WebMediaPlayerMS::decodedFrameCount() const {
    361   DCHECK(thread_checker_.CalledOnValidThread());
    362   DVLOG(1) << "WebMediaPlayerMS::decodedFrameCount, " << total_frame_count_;
    363   return total_frame_count_;
    364 }
    365 
    366 unsigned WebMediaPlayerMS::droppedFrameCount() const {
    367   DCHECK(thread_checker_.CalledOnValidThread());
    368   DVLOG(1) << "WebMediaPlayerMS::droppedFrameCount, " << dropped_frame_count_;
    369   return dropped_frame_count_;
    370 }
    371 
    372 unsigned WebMediaPlayerMS::audioDecodedByteCount() const {
    373   DCHECK(thread_checker_.CalledOnValidThread());
    374   NOTIMPLEMENTED();
    375   return 0;
    376 }
    377 
    378 unsigned WebMediaPlayerMS::videoDecodedByteCount() const {
    379   DCHECK(thread_checker_.CalledOnValidThread());
    380   NOTIMPLEMENTED();
    381   return 0;
    382 }
    383 
    384 void WebMediaPlayerMS::SetVideoFrameProviderClient(
    385     cc::VideoFrameProvider::Client* client) {
    386   // This is called from both the main renderer thread and the compositor
    387   // thread (when the main thread is blocked).
    388   if (video_frame_provider_client_)
    389     video_frame_provider_client_->StopUsingProvider();
    390   video_frame_provider_client_ = client;
    391 }
    392 
    393 scoped_refptr<media::VideoFrame> WebMediaPlayerMS::GetCurrentFrame() {
    394   DVLOG(3) << "WebMediaPlayerMS::GetCurrentFrame";
    395   base::AutoLock auto_lock(current_frame_lock_);
    396   DCHECK(!pending_repaint_);
    397   if (!current_frame_.get())
    398     return NULL;
    399   pending_repaint_ = true;
    400   current_frame_used_ = true;
    401   return current_frame_;
    402 }
    403 
    404 void WebMediaPlayerMS::PutCurrentFrame(
    405     const scoped_refptr<media::VideoFrame>& frame) {
    406   DVLOG(3) << "WebMediaPlayerMS::PutCurrentFrame";
    407   DCHECK(pending_repaint_);
    408   pending_repaint_ = false;
    409 }
    410 
    411 void WebMediaPlayerMS::OnFrameAvailable(
    412     const scoped_refptr<media::VideoFrame>& frame) {
    413   DVLOG(3) << "WebMediaPlayerMS::OnFrameAvailable";
    414   DCHECK(thread_checker_.CalledOnValidThread());
    415   ++total_frame_count_;
    416   if (!received_first_frame_) {
    417     received_first_frame_ = true;
    418     {
    419       base::AutoLock auto_lock(current_frame_lock_);
    420       DCHECK(!current_frame_used_);
    421       current_frame_ = frame;
    422     }
    423     SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
    424     SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
    425     GetClient()->sizeChanged();
    426 
    427     if (video_frame_provider_.get()) {
    428       video_weblayer_.reset(new cc_blink::WebLayerImpl(
    429           cc::VideoLayer::Create(this, media::VIDEO_ROTATION_0)));
    430       video_weblayer_->setOpaque(true);
    431       GetClient()->setWebLayer(video_weblayer_.get());
    432     }
    433   }
    434 
    435   // Do not update |current_frame_| when paused.
    436   if (paused_)
    437     return;
    438 
    439   bool size_changed = !current_frame_.get() ||
    440                       current_frame_->natural_size() != frame->natural_size();
    441 
    442   {
    443     base::AutoLock auto_lock(current_frame_lock_);
    444     if (!current_frame_used_ && current_frame_.get())
    445       ++dropped_frame_count_;
    446     current_frame_ = frame;
    447     current_time_ = frame->timestamp();
    448     current_frame_used_ = false;
    449   }
    450 
    451   if (size_changed)
    452     GetClient()->sizeChanged();
    453 
    454   GetClient()->repaint();
    455 }
    456 
    457 void WebMediaPlayerMS::RepaintInternal() {
    458   DVLOG(1) << "WebMediaPlayerMS::RepaintInternal";
    459   DCHECK(thread_checker_.CalledOnValidThread());
    460   GetClient()->repaint();
    461 }
    462 
    463 void WebMediaPlayerMS::OnSourceError() {
    464   DVLOG(1) << "WebMediaPlayerMS::OnSourceError";
    465   DCHECK(thread_checker_.CalledOnValidThread());
    466   SetNetworkState(WebMediaPlayer::NetworkStateFormatError);
    467   RepaintInternal();
    468 }
    469 
    470 void WebMediaPlayerMS::SetNetworkState(WebMediaPlayer::NetworkState state) {
    471   DCHECK(thread_checker_.CalledOnValidThread());
    472   network_state_ = state;
    473   // Always notify to ensure client has the latest value.
    474   GetClient()->networkStateChanged();
    475 }
    476 
    477 void WebMediaPlayerMS::SetReadyState(WebMediaPlayer::ReadyState state) {
    478   DCHECK(thread_checker_.CalledOnValidThread());
    479   ready_state_ = state;
    480   // Always notify to ensure client has the latest value.
    481   GetClient()->readyStateChanged();
    482 }
    483 
    484 blink::WebMediaPlayerClient* WebMediaPlayerMS::GetClient() {
    485   DCHECK(thread_checker_.CalledOnValidThread());
    486   DCHECK(client_);
    487   return client_;
    488 }
    489 
    490 }  // namespace content
    491