Home | History | Annotate | Download | only in resources 
      1 // Copyright 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "cc/resources/video_resource_updater.h"
      6 
      7 #include "base/bind.h"
      8 #include "cc/output/gl_renderer.h"
      9 #include "cc/resources/resource_provider.h"
     10 #include "gpu/GLES2/gl2extchromium.h"
     11 #include "gpu/command_buffer/client/gles2_interface.h"
     12 #include "media/base/video_frame.h"
     13 #include "media/filters/skcanvas_video_renderer.h"
     14 #include "third_party/khronos/GLES2/gl2.h"
     15 #include "third_party/khronos/GLES2/gl2ext.h"
     16 #include "ui/gfx/size_conversions.h"
     17 
     18 namespace cc {
     19 
     20 const ResourceFormat kYUVResourceFormat = LUMINANCE_8;
     21 const ResourceFormat kRGBResourceFormat = RGBA_8888;
     22 
     23 VideoFrameExternalResources::VideoFrameExternalResources() : type(NONE) {}
     24 
     25 VideoFrameExternalResources::~VideoFrameExternalResources() {}
     26 
     27 VideoResourceUpdater::VideoResourceUpdater(ContextProvider* context_provider,
     28                                            ResourceProvider* resource_provider)
     29     : context_provider_(context_provider),
     30       resource_provider_(resource_provider) {
     31 }
     32 
     33 VideoResourceUpdater::~VideoResourceUpdater() {
     34   while (!all_resources_.empty()) {
     35     resource_provider_->DeleteResource(all_resources_.back());
     36     all_resources_.pop_back();
     37   }
     38 }
     39 
     40 void VideoResourceUpdater::DeleteResource(unsigned resource_id) {
     41   resource_provider_->DeleteResource(resource_id);
     42   all_resources_.erase(std::remove(all_resources_.begin(),
     43                                    all_resources_.end(),
     44                                    resource_id));
     45 }
     46 
     47 VideoFrameExternalResources VideoResourceUpdater::
     48     CreateExternalResourcesFromVideoFrame(
     49         const scoped_refptr<media::VideoFrame>& video_frame) {
     50   if (!VerifyFrame(video_frame))
     51     return VideoFrameExternalResources();
     52 
     53   if (video_frame->format() == media::VideoFrame::NATIVE_TEXTURE)
     54     return CreateForHardwarePlanes(video_frame);
     55   else
     56     return CreateForSoftwarePlanes(video_frame);
     57 }
     58 
     59 bool VideoResourceUpdater::VerifyFrame(
     60     const scoped_refptr<media::VideoFrame>& video_frame) {
     61   // If these fail, we'll have to add logic that handles offset bitmap/texture
     62   // UVs. For now, just expect (0, 0) offset, since all our decoders so far
     63   // don't offset.
     64   DCHECK_EQ(video_frame->visible_rect().x(), 0);
     65   DCHECK_EQ(video_frame->visible_rect().y(), 0);
     66 
     67   switch (video_frame->format()) {
     68     // Acceptable inputs.
     69     case media::VideoFrame::YV12:
     70     case media::VideoFrame::YV12A:
     71     case media::VideoFrame::YV16:
     72     case media::VideoFrame::YV12J:
     73     case media::VideoFrame::NATIVE_TEXTURE:
     74 #if defined(VIDEO_HOLE)
     75     case media::VideoFrame::HOLE:
     76 #endif  // defined(VIDEO_HOLE)
     77       return true;
     78 
     79     // Unacceptable inputs. \(_o)/
     80     case media::VideoFrame::UNKNOWN:
     81     case media::VideoFrame::HISTOGRAM_MAX:
     82     case media::VideoFrame::I420:
     83       break;
     84   }
     85   return false;
     86 }
     87 
     88 // For frames that we receive in software format, determine the dimensions of
     89 // each plane in the frame.
     90 static gfx::Size SoftwarePlaneDimension(
     91     media::VideoFrame::Format input_frame_format,
     92     gfx::Size coded_size,
     93     ResourceFormat output_resource_format,
     94     int plane_index) {
     95   if (output_resource_format == kYUVResourceFormat) {
     96     if (plane_index == media::VideoFrame::kYPlane ||
     97         plane_index == media::VideoFrame::kAPlane)
     98       return coded_size;
     99 
    100     switch (input_frame_format) {
    101       case media::VideoFrame::YV12:
    102       case media::VideoFrame::YV12A:
    103       case media::VideoFrame::YV12J:
    104         return gfx::ToFlooredSize(gfx::ScaleSize(coded_size, 0.5f, 0.5f));
    105       case media::VideoFrame::YV16:
    106         return gfx::ToFlooredSize(gfx::ScaleSize(coded_size, 0.5f, 1.f));
    107 
    108       case media::VideoFrame::UNKNOWN:
    109       case media::VideoFrame::I420:
    110       case media::VideoFrame::NATIVE_TEXTURE:
    111       case media::VideoFrame::HISTOGRAM_MAX:
    112 #if defined(VIDEO_HOLE)
    113       case media::VideoFrame::HOLE:
    114 #endif  // defined(VIDEO_HOLE)
    115         NOTREACHED();
    116     }
    117   }
    118 
    119   DCHECK_EQ(output_resource_format, kRGBResourceFormat);
    120   return coded_size;
    121 }
    122 
    123 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
    124     const scoped_refptr<media::VideoFrame>& video_frame) {
    125   media::VideoFrame::Format input_frame_format = video_frame->format();
    126 
    127 #if defined(VIDEO_HOLE)
    128   if (input_frame_format == media::VideoFrame::HOLE) {
    129     VideoFrameExternalResources external_resources;
    130     external_resources.type = VideoFrameExternalResources::HOLE;
    131     return external_resources;
    132   }
    133 #endif  // defined(VIDEO_HOLE)
    134 
    135   // Only YUV software video frames are supported.
    136   DCHECK(input_frame_format == media::VideoFrame::YV12 ||
    137          input_frame_format == media::VideoFrame::YV12A ||
    138          input_frame_format == media::VideoFrame::YV12J ||
    139          input_frame_format == media::VideoFrame::YV16);
    140   if (input_frame_format != media::VideoFrame::YV12 &&
    141       input_frame_format != media::VideoFrame::YV12A &&
    142       input_frame_format != media::VideoFrame::YV12J &&
    143       input_frame_format != media::VideoFrame::YV16)
    144     return VideoFrameExternalResources();
    145 
    146   bool software_compositor = context_provider_ == NULL;
    147 
    148   ResourceFormat output_resource_format = kYUVResourceFormat;
    149   size_t output_plane_count =
    150       (input_frame_format == media::VideoFrame::YV12A) ? 4 : 3;
    151 
    152   // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
    153   // conversion here. That involves an extra copy of each frame to a bitmap.
    154   // Obviously, this is suboptimal and should be addressed once ubercompositor
    155   // starts shaping up.
    156   if (software_compositor) {
    157     output_resource_format = kRGBResourceFormat;
    158     output_plane_count = 1;
    159   }
    160 
    161   int max_resource_size = resource_provider_->max_texture_size();
    162   gfx::Size coded_frame_size = video_frame->coded_size();
    163 
    164   std::vector<PlaneResource> plane_resources;
    165   bool allocation_success = true;
    166 
    167   for (size_t i = 0; i < output_plane_count; ++i) {
    168     gfx::Size output_plane_resource_size =
    169         SoftwarePlaneDimension(input_frame_format,
    170                                coded_frame_size,
    171                                output_resource_format,
    172                                i);
    173     if (output_plane_resource_size.IsEmpty() ||
    174         output_plane_resource_size.width() > max_resource_size ||
    175         output_plane_resource_size.height() > max_resource_size) {
    176       allocation_success = false;
    177       break;
    178     }
    179 
    180     ResourceProvider::ResourceId resource_id = 0;
    181     gpu::Mailbox mailbox;
    182 
    183     // Try recycle a previously-allocated resource.
    184     for (size_t i = 0; i < recycled_resources_.size(); ++i) {
    185       if (recycled_resources_[i].resource_format == output_resource_format &&
    186           recycled_resources_[i].resource_size == output_plane_resource_size) {
    187         resource_id = recycled_resources_[i].resource_id;
    188         mailbox = recycled_resources_[i].mailbox;
    189         recycled_resources_.erase(recycled_resources_.begin() + i);
    190         break;
    191       }
    192     }
    193 
    194     if (resource_id == 0) {
    195       // TODO(danakj): Abstract out hw/sw resource create/delete from
    196       // ResourceProvider and stop using ResourceProvider in this class.
    197       resource_id =
    198           resource_provider_->CreateResource(output_plane_resource_size,
    199                                              GL_CLAMP_TO_EDGE,
    200                                              ResourceProvider::TextureUsageAny,
    201                                              output_resource_format);
    202 
    203       DCHECK(mailbox.IsZero());
    204 
    205       if (!software_compositor) {
    206         DCHECK(context_provider_);
    207 
    208         gpu::gles2::GLES2Interface* gl = context_provider_->ContextGL();
    209 
    210         GLC(gl, gl->GenMailboxCHROMIUM(mailbox.name));
    211         if (mailbox.IsZero()) {
    212           resource_provider_->DeleteResource(resource_id);
    213           resource_id = 0;
    214         } else {
    215           ResourceProvider::ScopedWriteLockGL lock(
    216               resource_provider_, resource_id);
    217           GLC(gl, gl->BindTexture(GL_TEXTURE_2D, lock.texture_id()));
    218           GLC(gl, gl->ProduceTextureCHROMIUM(GL_TEXTURE_2D, mailbox.name));
    219           GLC(gl, gl->BindTexture(GL_TEXTURE_2D, 0));
    220         }
    221       }
    222 
    223       if (resource_id)
    224         all_resources_.push_back(resource_id);
    225     }
    226 
    227     if (resource_id == 0) {
    228       allocation_success = false;
    229       break;
    230     }
    231 
    232     DCHECK(software_compositor || !mailbox.IsZero());
    233     plane_resources.push_back(PlaneResource(resource_id,
    234                                             output_plane_resource_size,
    235                                             output_resource_format,
    236                                             mailbox));
    237   }
    238 
    239   if (!allocation_success) {
    240     for (size_t i = 0; i < plane_resources.size(); ++i)
    241       DeleteResource(plane_resources[i].resource_id);
    242     return VideoFrameExternalResources();
    243   }
    244 
    245   VideoFrameExternalResources external_resources;
    246 
    247   if (software_compositor) {
    248     DCHECK_EQ(plane_resources.size(), 1u);
    249     DCHECK_EQ(plane_resources[0].resource_format, kRGBResourceFormat);
    250     DCHECK(plane_resources[0].mailbox.IsZero());
    251 
    252     if (!video_renderer_)
    253       video_renderer_.reset(new media::SkCanvasVideoRenderer);
    254 
    255     {
    256       ResourceProvider::ScopedWriteLockSoftware lock(
    257           resource_provider_, plane_resources[0].resource_id);
    258       video_renderer_->Paint(video_frame.get(),
    259                              lock.sk_canvas(),
    260                              video_frame->visible_rect(),
    261                              0xff);
    262     }
    263 
    264     RecycleResourceData recycle_data = {
    265       plane_resources[0].resource_id,
    266       plane_resources[0].resource_size,
    267       plane_resources[0].resource_format,
    268       gpu::Mailbox()
    269     };
    270     base::SharedMemory* shared_memory =
    271         resource_provider_->GetSharedMemory(plane_resources[0].resource_id);
    272     if (shared_memory) {
    273       external_resources.mailboxes.push_back(
    274           TextureMailbox(shared_memory, plane_resources[0].resource_size));
    275       external_resources.release_callbacks
    276           .push_back(base::Bind(&RecycleResource, AsWeakPtr(), recycle_data));
    277       external_resources.type = VideoFrameExternalResources::RGB_RESOURCE;
    278     } else {
    279       // TODO(jbauman): Remove this path once shared memory is available
    280       // everywhere.
    281       external_resources.software_resources
    282           .push_back(plane_resources[0].resource_id);
    283       external_resources.software_release_callback =
    284           base::Bind(&RecycleResource, AsWeakPtr(), recycle_data);
    285       external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
    286     }
    287 
    288     return external_resources;
    289   }
    290 
    291   for (size_t i = 0; i < plane_resources.size(); ++i) {
    292     // Update each plane's resource id with its content.
    293     DCHECK_EQ(plane_resources[i].resource_format, kYUVResourceFormat);
    294 
    295     const uint8_t* input_plane_pixels = video_frame->data(i);
    296 
    297     gfx::Rect image_rect(0,
    298                          0,
    299                          video_frame->stride(i),
    300                          plane_resources[i].resource_size.height());
    301     gfx::Rect source_rect(plane_resources[i].resource_size);
    302     resource_provider_->SetPixels(plane_resources[i].resource_id,
    303                                   input_plane_pixels,
    304                                   image_rect,
    305                                   source_rect,
    306                                   gfx::Vector2d());
    307 
    308     RecycleResourceData recycle_data = {
    309       plane_resources[i].resource_id,
    310       plane_resources[i].resource_size,
    311       plane_resources[i].resource_format,
    312       plane_resources[i].mailbox
    313     };
    314 
    315     external_resources.mailboxes.push_back(
    316         TextureMailbox(plane_resources[i].mailbox));
    317     external_resources.release_callbacks.push_back(
    318         base::Bind(&RecycleResource, AsWeakPtr(), recycle_data));
    319   }
    320 
    321   external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
    322   return external_resources;
    323 }
    324 
    325 static void ReturnTexture(const scoped_refptr<media::VideoFrame>& frame,
    326                           unsigned sync_point,
    327                           bool lost_resource) {
    328   frame->texture_mailbox()->Resync(sync_point);
    329 }
    330 
    331 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
    332     const scoped_refptr<media::VideoFrame>& video_frame) {
    333   media::VideoFrame::Format frame_format = video_frame->format();
    334 
    335   DCHECK_EQ(frame_format, media::VideoFrame::NATIVE_TEXTURE);
    336   if (frame_format != media::VideoFrame::NATIVE_TEXTURE)
    337       return VideoFrameExternalResources();
    338 
    339   if (!context_provider_)
    340     return VideoFrameExternalResources();
    341 
    342   VideoFrameExternalResources external_resources;
    343   switch (video_frame->texture_target()) {
    344     case GL_TEXTURE_2D:
    345       external_resources.type = VideoFrameExternalResources::RGB_RESOURCE;
    346       break;
    347     case GL_TEXTURE_EXTERNAL_OES:
    348       external_resources.type =
    349           VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
    350       break;
    351     case GL_TEXTURE_RECTANGLE_ARB:
    352       external_resources.type = VideoFrameExternalResources::IO_SURFACE;
    353       break;
    354     default:
    355       NOTREACHED();
    356       return VideoFrameExternalResources();
    357   }
    358 
    359   media::VideoFrame::MailboxHolder* mailbox_holder =
    360       video_frame->texture_mailbox();
    361 
    362   external_resources.mailboxes.push_back(
    363       TextureMailbox(mailbox_holder->mailbox(),
    364                      video_frame->texture_target(),
    365                      mailbox_holder->sync_point()));
    366   external_resources.release_callbacks.push_back(
    367       base::Bind(&ReturnTexture, video_frame));
    368   return external_resources;
    369 }
    370 
    371 // static
    372 void VideoResourceUpdater::RecycleResource(
    373     base::WeakPtr<VideoResourceUpdater> updater,
    374     RecycleResourceData data,
    375     unsigned sync_point,
    376     bool lost_resource) {
    377   if (!updater.get()) {
    378     // Resource was already deleted.
    379     return;
    380   }
    381 
    382   ContextProvider* context_provider = updater->context_provider_;
    383   if (context_provider && sync_point) {
    384     GLC(context_provider->ContextGL(),
    385         context_provider->ContextGL()->WaitSyncPointCHROMIUM(sync_point));
    386   }
    387 
    388   if (lost_resource) {
    389     updater->DeleteResource(data.resource_id);
    390     return;
    391   }
    392 
    393   // Drop recycled resources that are the wrong format.
    394   while (!updater->recycled_resources_.empty() &&
    395          updater->recycled_resources_.back().resource_format !=
    396          data.resource_format) {
    397     updater->DeleteResource(updater->recycled_resources_.back().resource_id);
    398     updater->recycled_resources_.pop_back();
    399   }
    400 
    401   PlaneResource recycled_resource(data.resource_id,
    402                                   data.resource_size,
    403                                   data.resource_format,
    404                                   data.mailbox);
    405   updater->recycled_resources_.push_back(recycled_resource);
    406 }
    407 
    408 }  // namespace cc
    409