Home | History | Annotate | Download | only in resources 
      1 // Copyright 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "cc/resources/video_resource_updater.h"
      6 
      7 #include "base/bind.h"
      8 #include "cc/output/gl_renderer.h"
      9 #include "cc/resources/resource_provider.h"
     10 #include "gpu/GLES2/gl2extchromium.h"
     11 #include "gpu/command_buffer/client/gles2_interface.h"
     12 #include "media/base/video_frame.h"
     13 #include "media/filters/skcanvas_video_renderer.h"
     14 #include "third_party/khronos/GLES2/gl2.h"
     15 #include "third_party/khronos/GLES2/gl2ext.h"
     16 #include "ui/gfx/size_conversions.h"
     17 
     18 namespace cc {
     19 
     20 const ResourceFormat kYUVResourceFormat = LUMINANCE_8;
     21 const ResourceFormat kRGBResourceFormat = RGBA_8888;
     22 
     23 VideoFrameExternalResources::VideoFrameExternalResources() : type(NONE) {}
     24 
     25 VideoFrameExternalResources::~VideoFrameExternalResources() {}
     26 
     27 VideoResourceUpdater::VideoResourceUpdater(ContextProvider* context_provider,
     28                                            ResourceProvider* resource_provider)
     29     : context_provider_(context_provider),
     30       resource_provider_(resource_provider) {
     31 }
     32 
     33 VideoResourceUpdater::~VideoResourceUpdater() {
     34   while (!all_resources_.empty()) {
     35     resource_provider_->DeleteResource(all_resources_.back());
     36     all_resources_.pop_back();
     37   }
     38 }
     39 
     40 void VideoResourceUpdater::DeleteResource(unsigned resource_id) {
     41   resource_provider_->DeleteResource(resource_id);
     42   all_resources_.erase(std::remove(all_resources_.begin(),
     43                                    all_resources_.end(),
     44                                    resource_id));
     45 }
     46 
     47 VideoFrameExternalResources VideoResourceUpdater::
     48     CreateExternalResourcesFromVideoFrame(
     49         const scoped_refptr<media::VideoFrame>& video_frame) {
     50   if (!VerifyFrame(video_frame))
     51     return VideoFrameExternalResources();
     52 
     53   if (video_frame->format() == media::VideoFrame::NATIVE_TEXTURE)
     54     return CreateForHardwarePlanes(video_frame);
     55   else
     56     return CreateForSoftwarePlanes(video_frame);
     57 }
     58 
     59 bool VideoResourceUpdater::VerifyFrame(
     60     const scoped_refptr<media::VideoFrame>& video_frame) {
     61   switch (video_frame->format()) {
     62     // Acceptable inputs.
     63     case media::VideoFrame::YV12:
     64     case media::VideoFrame::I420:
     65     case media::VideoFrame::YV12A:
     66     case media::VideoFrame::YV16:
     67     case media::VideoFrame::YV12J:
     68     case media::VideoFrame::YV24:
     69     case media::VideoFrame::NATIVE_TEXTURE:
     70 #if defined(VIDEO_HOLE)
     71     case media::VideoFrame::HOLE:
     72 #endif  // defined(VIDEO_HOLE)
     73       return true;
     74 
     75     // Unacceptable inputs. \(_o)/
     76     case media::VideoFrame::UNKNOWN:
     77     case media::VideoFrame::NV12:
     78       break;
     79   }
     80   return false;
     81 }
     82 
     83 // For frames that we receive in software format, determine the dimensions of
     84 // each plane in the frame.
     85 static gfx::Size SoftwarePlaneDimension(
     86     const scoped_refptr<media::VideoFrame>& input_frame,
     87     ResourceFormat output_resource_format,
     88     size_t plane_index) {
     89   if (output_resource_format == kYUVResourceFormat) {
     90     return media::VideoFrame::PlaneSize(
     91         input_frame->format(), plane_index, input_frame->coded_size());
     92   }
     93 
     94   DCHECK_EQ(output_resource_format, kRGBResourceFormat);
     95   return input_frame->coded_size();
     96 }
     97 
     98 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
     99     const scoped_refptr<media::VideoFrame>& video_frame) {
    100   media::VideoFrame::Format input_frame_format = video_frame->format();
    101 
    102 #if defined(VIDEO_HOLE)
    103   if (input_frame_format == media::VideoFrame::HOLE) {
    104     VideoFrameExternalResources external_resources;
    105     external_resources.type = VideoFrameExternalResources::HOLE;
    106     return external_resources;
    107   }
    108 #endif  // defined(VIDEO_HOLE)
    109 
    110   // Only YUV software video frames are supported.
    111   DCHECK(input_frame_format == media::VideoFrame::YV12 ||
    112          input_frame_format == media::VideoFrame::I420 ||
    113          input_frame_format == media::VideoFrame::YV12A ||
    114          input_frame_format == media::VideoFrame::YV12J ||
    115          input_frame_format == media::VideoFrame::YV16 ||
    116          input_frame_format == media::VideoFrame::YV24);
    117   if (input_frame_format != media::VideoFrame::YV12 &&
    118       input_frame_format != media::VideoFrame::I420 &&
    119       input_frame_format != media::VideoFrame::YV12A &&
    120       input_frame_format != media::VideoFrame::YV12J &&
    121       input_frame_format != media::VideoFrame::YV16 &&
    122       input_frame_format != media::VideoFrame::YV24)
    123     return VideoFrameExternalResources();
    124 
    125   bool software_compositor = context_provider_ == NULL;
    126 
    127   ResourceFormat output_resource_format = kYUVResourceFormat;
    128   size_t output_plane_count = media::VideoFrame::NumPlanes(input_frame_format);
    129 
    130   // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
    131   // conversion here. That involves an extra copy of each frame to a bitmap.
    132   // Obviously, this is suboptimal and should be addressed once ubercompositor
    133   // starts shaping up.
    134   if (software_compositor) {
    135     output_resource_format = kRGBResourceFormat;
    136     output_plane_count = 1;
    137   }
    138 
    139   int max_resource_size = resource_provider_->max_texture_size();
    140   std::vector<PlaneResource> plane_resources;
    141   bool allocation_success = true;
    142 
    143   for (size_t i = 0; i < output_plane_count; ++i) {
    144     gfx::Size output_plane_resource_size =
    145         SoftwarePlaneDimension(video_frame, output_resource_format, i);
    146     if (output_plane_resource_size.IsEmpty() ||
    147         output_plane_resource_size.width() > max_resource_size ||
    148         output_plane_resource_size.height() > max_resource_size) {
    149       allocation_success = false;
    150       break;
    151     }
    152 
    153     ResourceProvider::ResourceId resource_id = 0;
    154     gpu::Mailbox mailbox;
    155 
    156     // Try recycle a previously-allocated resource.
    157     for (size_t i = 0; i < recycled_resources_.size(); ++i) {
    158       bool resource_matches =
    159           recycled_resources_[i].resource_format == output_resource_format &&
    160           recycled_resources_[i].resource_size == output_plane_resource_size;
    161       bool not_in_use =
    162           !software_compositor || !resource_provider_->InUseByConsumer(
    163                                        recycled_resources_[i].resource_id);
    164       if (resource_matches && not_in_use) {
    165         resource_id = recycled_resources_[i].resource_id;
    166         mailbox = recycled_resources_[i].mailbox;
    167         recycled_resources_.erase(recycled_resources_.begin() + i);
    168         break;
    169       }
    170     }
    171 
    172     if (resource_id == 0) {
    173       // TODO(danakj): Abstract out hw/sw resource create/delete from
    174       // ResourceProvider and stop using ResourceProvider in this class.
    175       resource_id =
    176           resource_provider_->CreateResource(output_plane_resource_size,
    177                                              GL_CLAMP_TO_EDGE,
    178                                              ResourceProvider::TextureUsageAny,
    179                                              output_resource_format);
    180 
    181       DCHECK(mailbox.IsZero());
    182 
    183       if (!software_compositor) {
    184         DCHECK(context_provider_);
    185 
    186         gpu::gles2::GLES2Interface* gl = context_provider_->ContextGL();
    187 
    188         GLC(gl, gl->GenMailboxCHROMIUM(mailbox.name));
    189         ResourceProvider::ScopedWriteLockGL lock(resource_provider_,
    190                                                  resource_id);
    191         GLC(gl, gl->BindTexture(GL_TEXTURE_2D, lock.texture_id()));
    192         GLC(gl, gl->ProduceTextureCHROMIUM(GL_TEXTURE_2D, mailbox.name));
    193         GLC(gl, gl->BindTexture(GL_TEXTURE_2D, 0));
    194       }
    195 
    196       if (resource_id)
    197         all_resources_.push_back(resource_id);
    198     }
    199 
    200     if (resource_id == 0) {
    201       allocation_success = false;
    202       break;
    203     }
    204 
    205     DCHECK(software_compositor || !mailbox.IsZero());
    206     plane_resources.push_back(PlaneResource(resource_id,
    207                                             output_plane_resource_size,
    208                                             output_resource_format,
    209                                             mailbox));
    210   }
    211 
    212   if (!allocation_success) {
    213     for (size_t i = 0; i < plane_resources.size(); ++i)
    214       DeleteResource(plane_resources[i].resource_id);
    215     return VideoFrameExternalResources();
    216   }
    217 
    218   VideoFrameExternalResources external_resources;
    219 
    220   if (software_compositor) {
    221     DCHECK_EQ(plane_resources.size(), 1u);
    222     DCHECK_EQ(plane_resources[0].resource_format, kRGBResourceFormat);
    223     DCHECK(plane_resources[0].mailbox.IsZero());
    224 
    225     if (!video_renderer_)
    226       video_renderer_.reset(new media::SkCanvasVideoRenderer);
    227 
    228     {
    229       ResourceProvider::ScopedWriteLockSoftware lock(
    230           resource_provider_, plane_resources[0].resource_id);
    231       video_renderer_->Paint(video_frame.get(),
    232                              lock.sk_canvas(),
    233                              video_frame->visible_rect(),
    234                              0xff);
    235     }
    236 
    237     RecycleResourceData recycle_data = {
    238       plane_resources[0].resource_id,
    239       plane_resources[0].resource_size,
    240       plane_resources[0].resource_format,
    241       gpu::Mailbox()
    242     };
    243     external_resources.software_resources.push_back(
    244         plane_resources[0].resource_id);
    245     external_resources.software_release_callback =
    246         base::Bind(&RecycleResource, AsWeakPtr(), recycle_data);
    247     external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
    248 
    249     return external_resources;
    250   }
    251 
    252   for (size_t i = 0; i < plane_resources.size(); ++i) {
    253     // Update each plane's resource id with its content.
    254     DCHECK_EQ(plane_resources[i].resource_format, kYUVResourceFormat);
    255 
    256     const uint8_t* input_plane_pixels = video_frame->data(i);
    257 
    258     gfx::Rect image_rect(0,
    259                          0,
    260                          video_frame->stride(i),
    261                          plane_resources[i].resource_size.height());
    262     gfx::Rect source_rect(plane_resources[i].resource_size);
    263     resource_provider_->SetPixels(plane_resources[i].resource_id,
    264                                   input_plane_pixels,
    265                                   image_rect,
    266                                   source_rect,
    267                                   gfx::Vector2d());
    268 
    269     RecycleResourceData recycle_data = {
    270       plane_resources[i].resource_id,
    271       plane_resources[i].resource_size,
    272       plane_resources[i].resource_format,
    273       plane_resources[i].mailbox
    274     };
    275 
    276     external_resources.mailboxes.push_back(
    277         TextureMailbox(plane_resources[i].mailbox, GL_TEXTURE_2D, 0));
    278     external_resources.release_callbacks.push_back(
    279         base::Bind(&RecycleResource, AsWeakPtr(), recycle_data));
    280   }
    281 
    282   external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
    283   return external_resources;
    284 }
    285 
    286 static void ReturnTexture(const scoped_refptr<media::VideoFrame>& frame,
    287                           uint32 sync_point,
    288                           bool lost_resource) {
    289   frame->AppendReleaseSyncPoint(sync_point);
    290 }
    291 
    292 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
    293     const scoped_refptr<media::VideoFrame>& video_frame) {
    294   media::VideoFrame::Format frame_format = video_frame->format();
    295 
    296   DCHECK_EQ(frame_format, media::VideoFrame::NATIVE_TEXTURE);
    297   if (frame_format != media::VideoFrame::NATIVE_TEXTURE)
    298       return VideoFrameExternalResources();
    299 
    300   if (!context_provider_)
    301     return VideoFrameExternalResources();
    302 
    303   const gpu::MailboxHolder* mailbox_holder = video_frame->mailbox_holder();
    304   VideoFrameExternalResources external_resources;
    305   switch (mailbox_holder->texture_target) {
    306     case GL_TEXTURE_2D:
    307       external_resources.type = VideoFrameExternalResources::RGB_RESOURCE;
    308       break;
    309     case GL_TEXTURE_EXTERNAL_OES:
    310       external_resources.type =
    311           VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
    312       break;
    313     case GL_TEXTURE_RECTANGLE_ARB:
    314       external_resources.type = VideoFrameExternalResources::IO_SURFACE;
    315       break;
    316     default:
    317       NOTREACHED();
    318       return VideoFrameExternalResources();
    319   }
    320 
    321   external_resources.mailboxes.push_back(
    322       TextureMailbox(mailbox_holder->mailbox,
    323                      mailbox_holder->texture_target,
    324                      mailbox_holder->sync_point));
    325   external_resources.release_callbacks.push_back(
    326       base::Bind(&ReturnTexture, video_frame));
    327   return external_resources;
    328 }
    329 
    330 // static
    331 void VideoResourceUpdater::RecycleResource(
    332     base::WeakPtr<VideoResourceUpdater> updater,
    333     RecycleResourceData data,
    334     uint32 sync_point,
    335     bool lost_resource) {
    336   if (!updater.get()) {
    337     // Resource was already deleted.
    338     return;
    339   }
    340 
    341   ContextProvider* context_provider = updater->context_provider_;
    342   if (context_provider && sync_point) {
    343     GLC(context_provider->ContextGL(),
    344         context_provider->ContextGL()->WaitSyncPointCHROMIUM(sync_point));
    345   }
    346 
    347   if (lost_resource) {
    348     updater->DeleteResource(data.resource_id);
    349     return;
    350   }
    351 
    352   // Drop recycled resources that are the wrong format.
    353   while (!updater->recycled_resources_.empty() &&
    354          updater->recycled_resources_.back().resource_format !=
    355          data.resource_format) {
    356     updater->DeleteResource(updater->recycled_resources_.back().resource_id);
    357     updater->recycled_resources_.pop_back();
    358   }
    359 
    360   PlaneResource recycled_resource(data.resource_id,
    361                                   data.resource_size,
    362                                   data.resource_format,
    363                                   data.mailbox);
    364   updater->recycled_resources_.push_back(recycled_resource);
    365 }
    366 
    367 }  // namespace cc
    368