Home | History | Annotate | Download | only in resources 
      1 // Copyright 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "cc/resources/video_resource_updater.h"
      6 
      7 #include "base/bind.h"
      8 #include "cc/output/gl_renderer.h"
      9 #include "cc/resources/resource_provider.h"
     10 #include "gpu/GLES2/gl2extchromium.h"
     11 #include "media/base/video_frame.h"
     12 #include "media/filters/skcanvas_video_renderer.h"
     13 #include "third_party/khronos/GLES2/gl2.h"
     14 #include "third_party/khronos/GLES2/gl2ext.h"
     15 #include "ui/gfx/size_conversions.h"
     16 
     17 const unsigned kYUVResourceFormat = GL_LUMINANCE;
     18 const unsigned kRGBResourceFormat = GL_RGBA;
     19 
     20 namespace cc {
     21 
     22 VideoFrameExternalResources::VideoFrameExternalResources() : type(NONE) {}
     23 
     24 VideoFrameExternalResources::~VideoFrameExternalResources() {}
     25 
     26 VideoResourceUpdater::VideoResourceUpdater(ResourceProvider* resource_provider)
     27     : resource_provider_(resource_provider) {
     28 }
     29 
     30 VideoResourceUpdater::~VideoResourceUpdater() {
     31   while (!all_resources_.empty()) {
     32     resource_provider_->DeleteResource(all_resources_.back());
     33     all_resources_.pop_back();
     34   }
     35 }
     36 
     37 void VideoResourceUpdater::DeleteResource(unsigned resource_id) {
     38   resource_provider_->DeleteResource(resource_id);
     39   all_resources_.erase(std::remove(all_resources_.begin(),
     40                                    all_resources_.end(),
     41                                    resource_id));
     42 }
     43 
     44 VideoFrameExternalResources VideoResourceUpdater::
     45     CreateExternalResourcesFromVideoFrame(
     46         const scoped_refptr<media::VideoFrame>& video_frame) {
     47   if (!VerifyFrame(video_frame))
     48     return VideoFrameExternalResources();
     49 
     50   if (video_frame->format() == media::VideoFrame::NATIVE_TEXTURE)
     51     return CreateForHardwarePlanes(video_frame);
     52   else
     53     return CreateForSoftwarePlanes(video_frame);
     54 }
     55 
     56 bool VideoResourceUpdater::VerifyFrame(
     57     const scoped_refptr<media::VideoFrame>& video_frame) {
     58   // If these fail, we'll have to add logic that handles offset bitmap/texture
     59   // UVs. For now, just expect (0, 0) offset, since all our decoders so far
     60   // don't offset.
     61   DCHECK_EQ(video_frame->visible_rect().x(), 0);
     62   DCHECK_EQ(video_frame->visible_rect().y(), 0);
     63 
     64   switch (video_frame->format()) {
     65     // Acceptable inputs.
     66     case media::VideoFrame::YV12:
     67     case media::VideoFrame::YV12A:
     68     case media::VideoFrame::YV16:
     69     case media::VideoFrame::NATIVE_TEXTURE:
     70 #if defined(GOOGLE_TV)
     71     case media::VideoFrame::HOLE:
     72 #endif
     73       return true;
     74 
     75     // Unacceptable inputs. \(_o)/
     76     case media::VideoFrame::INVALID:
     77     case media::VideoFrame::RGB32:
     78     case media::VideoFrame::EMPTY:
     79     case media::VideoFrame::I420:
     80       break;
     81   }
     82   return false;
     83 }
     84 
     85 // For frames that we receive in software format, determine the dimensions of
     86 // each plane in the frame.
     87 static gfx::Size SoftwarePlaneDimension(
     88     media::VideoFrame::Format input_frame_format,
     89     gfx::Size coded_size,
     90     GLenum output_resource_format,
     91     int plane_index) {
     92   if (output_resource_format == kYUVResourceFormat) {
     93     if (plane_index == media::VideoFrame::kYPlane ||
     94         plane_index == media::VideoFrame::kAPlane)
     95       return coded_size;
     96 
     97     switch (input_frame_format) {
     98       case media::VideoFrame::YV12:
     99       case media::VideoFrame::YV12A:
    100         return gfx::ToFlooredSize(gfx::ScaleSize(coded_size, 0.5f, 0.5f));
    101       case media::VideoFrame::YV16:
    102         return gfx::ToFlooredSize(gfx::ScaleSize(coded_size, 0.5f, 1.f));
    103 
    104       case media::VideoFrame::INVALID:
    105       case media::VideoFrame::RGB32:
    106       case media::VideoFrame::EMPTY:
    107       case media::VideoFrame::I420:
    108       case media::VideoFrame::NATIVE_TEXTURE:
    109 #if defined(GOOGLE_TV)
    110       case media::VideoFrame::HOLE:
    111 #endif
    112         NOTREACHED();
    113     }
    114   }
    115 
    116   DCHECK_EQ(output_resource_format, static_cast<unsigned>(kRGBResourceFormat));
    117   return coded_size;
    118 }
    119 
    120 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
    121     const scoped_refptr<media::VideoFrame>& video_frame) {
    122   media::VideoFrame::Format input_frame_format = video_frame->format();
    123 
    124 #if defined(GOOGLE_TV)
    125   if (input_frame_format == media::VideoFrame::HOLE) {
    126     VideoFrameExternalResources external_resources;
    127     external_resources.type = VideoFrameExternalResources::HOLE;
    128     return external_resources;
    129   }
    130 #endif
    131 
    132   // Only YUV software video frames are supported.
    133   DCHECK(input_frame_format == media::VideoFrame::YV12 ||
    134          input_frame_format == media::VideoFrame::YV12A ||
    135          input_frame_format == media::VideoFrame::YV16);
    136   if (input_frame_format != media::VideoFrame::YV12 &&
    137       input_frame_format != media::VideoFrame::YV12A &&
    138       input_frame_format != media::VideoFrame::YV16)
    139     return VideoFrameExternalResources();
    140 
    141   bool software_compositor = !resource_provider_->GraphicsContext3D();
    142 
    143   GLenum output_resource_format = kYUVResourceFormat;
    144   size_t output_plane_count =
    145       (input_frame_format == media::VideoFrame::YV12A) ? 4 : 3;
    146 
    147   // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
    148   // conversion here. That involves an extra copy of each frame to a bitmap.
    149   // Obviously, this is suboptimal and should be addressed once ubercompositor
    150   // starts shaping up.
    151   if (software_compositor) {
    152     output_resource_format = kRGBResourceFormat;
    153     output_plane_count = 1;
    154   }
    155 
    156   int max_resource_size = resource_provider_->max_texture_size();
    157   gfx::Size coded_frame_size = video_frame->coded_size();
    158 
    159   std::vector<PlaneResource> plane_resources;
    160   bool allocation_success = true;
    161 
    162   for (size_t i = 0; i < output_plane_count; ++i) {
    163     gfx::Size output_plane_resource_size =
    164         SoftwarePlaneDimension(input_frame_format,
    165                                coded_frame_size,
    166                                output_resource_format,
    167                                i);
    168     if (output_plane_resource_size.IsEmpty() ||
    169         output_plane_resource_size.width() > max_resource_size ||
    170         output_plane_resource_size.height() > max_resource_size) {
    171       allocation_success = false;
    172       break;
    173     }
    174 
    175     ResourceProvider::ResourceId resource_id = 0;
    176     gpu::Mailbox mailbox;
    177 
    178     // Try recycle a previously-allocated resource.
    179     for (size_t i = 0; i < recycled_resources_.size(); ++i) {
    180       if (recycled_resources_[i].resource_format == output_resource_format &&
    181           recycled_resources_[i].resource_size == output_plane_resource_size) {
    182         resource_id = recycled_resources_[i].resource_id;
    183         mailbox = recycled_resources_[i].mailbox;
    184         recycled_resources_.erase(recycled_resources_.begin() + i);
    185         break;
    186       }
    187     }
    188 
    189     if (resource_id == 0) {
    190       // TODO(danakj): Abstract out hw/sw resource create/delete from
    191       // ResourceProvider and stop using ResourceProvider in this class.
    192       resource_id =
    193           resource_provider_->CreateResource(output_plane_resource_size,
    194                                              output_resource_format,
    195                                              ResourceProvider::TextureUsageAny);
    196 
    197       DCHECK(mailbox.IsZero());
    198 
    199       if (!software_compositor) {
    200         WebKit::WebGraphicsContext3D* context =
    201             resource_provider_->GraphicsContext3D();
    202         DCHECK(context);
    203 
    204         GLC(context, context->genMailboxCHROMIUM(mailbox.name));
    205         if (mailbox.IsZero()) {
    206           resource_provider_->DeleteResource(resource_id);
    207           resource_id = 0;
    208         } else {
    209           ResourceProvider::ScopedWriteLockGL lock(
    210               resource_provider_, resource_id);
    211           GLC(context, context->bindTexture(GL_TEXTURE_2D, lock.texture_id()));
    212           GLC(context, context->produceTextureCHROMIUM(GL_TEXTURE_2D,
    213                                                        mailbox.name));
    214           GLC(context, context->bindTexture(GL_TEXTURE_2D, 0));
    215         }
    216       }
    217 
    218       if (resource_id)
    219         all_resources_.push_back(resource_id);
    220     }
    221 
    222     if (resource_id == 0) {
    223       allocation_success = false;
    224       break;
    225     }
    226 
    227     DCHECK(software_compositor || !mailbox.IsZero());
    228     plane_resources.push_back(PlaneResource(resource_id,
    229                                             output_plane_resource_size,
    230                                             output_resource_format,
    231                                             mailbox));
    232   }
    233 
    234   if (!allocation_success) {
    235     for (size_t i = 0; i < plane_resources.size(); ++i)
    236       DeleteResource(plane_resources[i].resource_id);
    237     return VideoFrameExternalResources();
    238   }
    239 
    240   VideoFrameExternalResources external_resources;
    241 
    242   if (software_compositor) {
    243     DCHECK_EQ(plane_resources.size(), 1u);
    244     DCHECK_EQ(plane_resources[0].resource_format, kRGBResourceFormat);
    245     DCHECK(plane_resources[0].mailbox.IsZero());
    246 
    247     if (!video_renderer_)
    248       video_renderer_.reset(new media::SkCanvasVideoRenderer);
    249 
    250     {
    251       ResourceProvider::ScopedWriteLockSoftware lock(
    252           resource_provider_, plane_resources[0].resource_id);
    253       video_renderer_->Paint(video_frame.get(),
    254                              lock.sk_canvas(),
    255                              video_frame->visible_rect(),
    256                              0xff);
    257     }
    258 
    259     // In software mode, the resource provider won't be lost. Soon this callback
    260     // will be called directly from the resource provider, same as 3d
    261     // compositing mode, so this raw unretained resource_provider will always
    262     // be valid when the callback is fired.
    263     RecycleResourceData recycle_data = {
    264       plane_resources[0].resource_id,
    265       plane_resources[0].resource_size,
    266       plane_resources[0].resource_format,
    267       gpu::Mailbox()
    268     };
    269     TextureMailbox::ReleaseCallback callback_to_free_resource =
    270         base::Bind(&RecycleResource,
    271                    AsWeakPtr(),
    272                    recycle_data);
    273     external_resources.software_resources.push_back(
    274         plane_resources[0].resource_id);
    275     external_resources.software_release_callback = callback_to_free_resource;
    276 
    277     external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
    278     return external_resources;
    279   }
    280 
    281   for (size_t i = 0; i < plane_resources.size(); ++i) {
    282     // Update each plane's resource id with its content.
    283     DCHECK_EQ(plane_resources[i].resource_format,
    284               static_cast<unsigned>(kYUVResourceFormat));
    285 
    286     const uint8_t* input_plane_pixels = video_frame->data(i);
    287 
    288     gfx::Rect image_rect(0,
    289                          0,
    290                          video_frame->stride(i),
    291                          plane_resources[i].resource_size.height());
    292     gfx::Rect source_rect(plane_resources[i].resource_size);
    293     resource_provider_->SetPixels(plane_resources[i].resource_id,
    294                                   input_plane_pixels,
    295                                   image_rect,
    296                                   source_rect,
    297                                   gfx::Vector2d());
    298 
    299     RecycleResourceData recycle_data = {
    300       plane_resources[i].resource_id,
    301       plane_resources[i].resource_size,
    302       plane_resources[i].resource_format,
    303       plane_resources[i].mailbox
    304     };
    305     TextureMailbox::ReleaseCallback callback_to_free_resource =
    306         base::Bind(&RecycleResource,
    307                    AsWeakPtr(),
    308                    recycle_data);
    309     external_resources.mailboxes.push_back(
    310         TextureMailbox(plane_resources[i].mailbox,
    311                        callback_to_free_resource));
    312   }
    313 
    314   external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
    315   return external_resources;
    316 }
    317 
    318 static void ReturnTexture(
    319     scoped_refptr<media::VideoFrame::MailboxHolder> mailbox_holder,
    320     unsigned sync_point,
    321     bool lost_resource) {
    322   mailbox_holder->Return(sync_point);
    323 }
    324 
    325 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
    326     const scoped_refptr<media::VideoFrame>& video_frame) {
    327   media::VideoFrame::Format frame_format = video_frame->format();
    328 
    329   DCHECK_EQ(frame_format, media::VideoFrame::NATIVE_TEXTURE);
    330   if (frame_format != media::VideoFrame::NATIVE_TEXTURE)
    331       return VideoFrameExternalResources();
    332 
    333   WebKit::WebGraphicsContext3D* context =
    334       resource_provider_->GraphicsContext3D();
    335   if (!context)
    336     return VideoFrameExternalResources();
    337 
    338   VideoFrameExternalResources external_resources;
    339   switch (video_frame->texture_target()) {
    340     case GL_TEXTURE_2D:
    341       external_resources.type = VideoFrameExternalResources::RGB_RESOURCE;
    342       break;
    343     case GL_TEXTURE_EXTERNAL_OES:
    344       external_resources.type =
    345           VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
    346       break;
    347     case GL_TEXTURE_RECTANGLE_ARB:
    348       external_resources.type = VideoFrameExternalResources::IO_SURFACE;
    349       break;
    350     default:
    351       NOTREACHED();
    352       return VideoFrameExternalResources();
    353   }
    354 
    355   scoped_refptr<media::VideoFrame::MailboxHolder> mailbox_holder =
    356       video_frame->texture_mailbox();
    357 
    358   TextureMailbox::ReleaseCallback callback_to_return_resource =
    359       base::Bind(&ReturnTexture, mailbox_holder);
    360 
    361   external_resources.mailboxes.push_back(
    362       TextureMailbox(mailbox_holder->mailbox(),
    363                      callback_to_return_resource,
    364                      video_frame->texture_target(),
    365                      mailbox_holder->sync_point()));
    366   return external_resources;
    367 }
    368 
    369 // static
    370 void VideoResourceUpdater::RecycleResource(
    371     base::WeakPtr<VideoResourceUpdater> updater,
    372     RecycleResourceData data,
    373     unsigned sync_point,
    374     bool lost_resource) {
    375   if (!updater.get()) {
    376     // Resource was already deleted.
    377     return;
    378   }
    379 
    380   WebKit::WebGraphicsContext3D* context =
    381       updater->resource_provider_->GraphicsContext3D();
    382   if (context && sync_point)
    383     GLC(context, context->waitSyncPoint(sync_point));
    384 
    385   if (lost_resource) {
    386     updater->DeleteResource(data.resource_id);
    387     return;
    388   }
    389 
    390   // Drop recycled resources that are the wrong format.
    391   while (!updater->recycled_resources_.empty() &&
    392          updater->recycled_resources_.back().resource_format !=
    393          data.resource_format) {
    394     updater->DeleteResource(updater->recycled_resources_.back().resource_id);
    395     updater->recycled_resources_.pop_back();
    396   }
    397 
    398   PlaneResource recycled_resource(data.resource_id,
    399                                   data.resource_size,
    400                                   data.resource_format,
    401                                   data.mailbox);
    402   updater->recycled_resources_.push_back(recycled_resource);
    403 }
    404 
    405 }  // namespace cc
    406