Home | History | Annotate | Download | only in plugin 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "remoting/client/plugin/pepper_view.h"
      6 
      7 #include <functional>
      8 
      9 #include "base/message_loop/message_loop.h"
     10 #include "base/strings/string_util.h"
     11 #include "base/synchronization/waitable_event.h"
     12 #include "base/time/time.h"
     13 #include "ppapi/cpp/completion_callback.h"
     14 #include "ppapi/cpp/image_data.h"
     15 #include "ppapi/cpp/point.h"
     16 #include "ppapi/cpp/rect.h"
     17 #include "ppapi/cpp/size.h"
     18 #include "remoting/base/util.h"
     19 #include "remoting/client/chromoting_stats.h"
     20 #include "remoting/client/client_context.h"
     21 #include "remoting/client/frame_producer.h"
     22 #include "remoting/client/plugin/chromoting_instance.h"
     23 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
     24 
     25 using base::Passed;
     26 
     27 namespace {
     28 
     29 // DesktopFrame that wraps a supplied pp::ImageData
     30 class PepperDesktopFrame : public webrtc::DesktopFrame {
     31  public:
     32   // Wraps the supplied ImageData.
     33   explicit PepperDesktopFrame(const pp::ImageData& buffer);
     34 
     35   // Access to underlying pepper representation.
     36   const pp::ImageData& buffer() const {
     37     return buffer_;
     38   }
     39 
     40  private:
     41   pp::ImageData buffer_;
     42 };
     43 
     44 PepperDesktopFrame::PepperDesktopFrame(const pp::ImageData& buffer)
     45   : DesktopFrame(webrtc::DesktopSize(buffer.size().width(),
     46                                      buffer.size().height()),
     47                  buffer.stride(),
     48                  reinterpret_cast<uint8_t*>(buffer.data()),
     49                  NULL),
     50     buffer_(buffer) {}
     51 
     52 }  // namespace
     53 
     54 namespace remoting {
     55 
     56 namespace {
     57 
     58 // The maximum number of image buffers to be allocated at any point of time.
     59 const size_t kMaxPendingBuffersCount = 2;
     60 
     61 }  // namespace
     62 
     63 PepperView::PepperView(ChromotingInstance* instance,
     64                        ClientContext* context)
     65   : instance_(instance),
     66     context_(context),
     67     producer_(NULL),
     68     merge_buffer_(NULL),
     69     dips_to_device_scale_(1.0f),
     70     dips_to_view_scale_(1.0f),
     71     flush_pending_(false),
     72     is_initialized_(false),
     73     frame_received_(false),
     74     callback_factory_(this) {
     75 }
     76 
     77 PepperView::~PepperView() {
     78   // The producer should now return any pending buffers. At this point, however,
     79   // ReturnBuffer() tasks scheduled by the producer will not be delivered,
     80   // so we free all the buffers once the producer's queue is empty.
     81   base::WaitableEvent done_event(true, false);
     82   producer_->RequestReturnBuffers(
     83       base::Bind(&base::WaitableEvent::Signal, base::Unretained(&done_event)));
     84   done_event.Wait();
     85 
     86   merge_buffer_ = NULL;
     87   while (!buffers_.empty()) {
     88     FreeBuffer(buffers_.front());
     89   }
     90 }
     91 
     92 void PepperView::Initialize(FrameProducer* producer) {
     93   producer_ = producer;
     94   webrtc::DesktopFrame* buffer = AllocateBuffer();
     95   while (buffer) {
     96     producer_->DrawBuffer(buffer);
     97     buffer = AllocateBuffer();
     98   }
     99 }
    100 
    101 void PepperView::SetView(const pp::View& view) {
    102   bool view_changed = false;
    103 
    104   pp::Rect pp_size = view.GetRect();
    105   webrtc::DesktopSize new_dips_size(pp_size.width(), pp_size.height());
    106   float new_dips_to_device_scale = view.GetDeviceScale();
    107 
    108   if (!dips_size_.equals(new_dips_size) ||
    109       dips_to_device_scale_ != new_dips_to_device_scale) {
    110     view_changed = true;
    111     dips_to_device_scale_ = new_dips_to_device_scale;
    112     dips_size_ = new_dips_size;
    113 
    114     // If |dips_to_device_scale_| is > 1.0 then the device is high-DPI, and
    115     // there are actually |view_device_scale_| physical pixels for every one
    116     // Density Independent Pixel (DIP).  If we specify a scale of 1.0 to
    117     // Graphics2D then we can render at DIP resolution and let PPAPI up-scale
    118     // for high-DPI devices.
    119     dips_to_view_scale_ = 1.0f;
    120     view_size_ = dips_size_;
    121 
    122     // If the view's DIP dimensions don't match the source then let the frame
    123     // producer do the scaling, and render at device resolution.
    124     if (!dips_size_.equals(source_size_)) {
    125       dips_to_view_scale_ = dips_to_device_scale_;
    126       view_size_.set(ceilf(dips_size_.width() * dips_to_view_scale_),
    127                      ceilf(dips_size_.height() * dips_to_view_scale_));
    128     }
    129 
    130     // Create a 2D rendering context at the chosen frame dimensions.
    131     pp::Size pp_size = pp::Size(view_size_.width(), view_size_.height());
    132     graphics2d_ = pp::Graphics2D(instance_, pp_size, false);
    133 
    134     // Specify the scale from our coordinates to DIPs.
    135     graphics2d_.SetScale(1.0f / dips_to_view_scale_);
    136 
    137     bool result = instance_->BindGraphics(graphics2d_);
    138 
    139     // There is no good way to handle this error currently.
    140     DCHECK(result) << "Couldn't bind the device context.";
    141   }
    142 
    143   pp::Rect pp_clip = view.GetClipRect();
    144   webrtc::DesktopRect new_clip = webrtc::DesktopRect::MakeLTRB(
    145       floorf(pp_clip.x() * dips_to_view_scale_),
    146       floorf(pp_clip.y() * dips_to_view_scale_),
    147       ceilf(pp_clip.right() * dips_to_view_scale_),
    148       ceilf(pp_clip.bottom() * dips_to_view_scale_));
    149   if (!clip_area_.equals(new_clip)) {
    150     view_changed = true;
    151 
    152     // YUV to RGB conversion may require even X and Y coordinates for
    153     // the top left corner of the clipping area.
    154     clip_area_ = AlignRect(new_clip);
    155     clip_area_.IntersectWith(webrtc::DesktopRect::MakeSize(view_size_));
    156   }
    157 
    158   if (view_changed) {
    159     producer_->SetOutputSizeAndClip(view_size_, clip_area_);
    160     Initialize(producer_);
    161   }
    162 }
    163 
    164 void PepperView::ApplyBuffer(const webrtc::DesktopSize& view_size,
    165                              const webrtc::DesktopRect& clip_area,
    166                              webrtc::DesktopFrame* buffer,
    167                              const webrtc::DesktopRegion& region) {
    168   DCHECK(context_->main_task_runner()->BelongsToCurrentThread());
    169 
    170   if (!frame_received_) {
    171     instance_->OnFirstFrameReceived();
    172     frame_received_ = true;
    173   }
    174   // We cannot use the data in the buffer if its dimensions don't match the
    175   // current view size.
    176   // TODO(alexeypa): We could rescale and draw it (or even draw it without
    177   // rescaling) to reduce the perceived lag while we are waiting for
    178   // the properly scaled data.
    179   if (!view_size_.equals(view_size)) {
    180     FreeBuffer(buffer);
    181     Initialize(producer_);
    182   } else {
    183     FlushBuffer(clip_area, buffer, region);
    184   }
    185 }
    186 
    187 void PepperView::ReturnBuffer(webrtc::DesktopFrame* buffer) {
    188   DCHECK(context_->main_task_runner()->BelongsToCurrentThread());
    189 
    190   // Reuse the buffer if it is large enough, otherwise drop it on the floor
    191   // and allocate a new one.
    192   if (buffer->size().width() >= clip_area_.width() &&
    193       buffer->size().height() >= clip_area_.height()) {
    194     producer_->DrawBuffer(buffer);
    195   } else {
    196     FreeBuffer(buffer);
    197     Initialize(producer_);
    198   }
    199 }
    200 
    201 void PepperView::SetSourceSize(const webrtc::DesktopSize& source_size,
    202                                const webrtc::DesktopVector& source_dpi) {
    203   DCHECK(context_->main_task_runner()->BelongsToCurrentThread());
    204 
    205   if (source_size_.equals(source_size) && source_dpi_.equals(source_dpi))
    206     return;
    207 
    208   source_size_ = source_size;
    209   source_dpi_ = source_dpi;
    210 
    211   // Notify JavaScript of the change in source size.
    212   instance_->SetDesktopSize(source_size, source_dpi);
    213 }
    214 
    215 FrameConsumer::PixelFormat PepperView::GetPixelFormat() {
    216   return FORMAT_BGRA;
    217 }
    218 
    219 webrtc::DesktopFrame* PepperView::AllocateBuffer() {
    220   if (buffers_.size() >= kMaxPendingBuffersCount)
    221     return NULL;
    222 
    223   if (clip_area_.width()==0 || clip_area_.height()==0)
    224     return NULL;
    225 
    226   // Create an image buffer of the required size, but don't zero it.
    227   pp::ImageData buffer_data(instance_,
    228                     PP_IMAGEDATAFORMAT_BGRA_PREMUL,
    229                     pp::Size(clip_area_.width(),
    230                              clip_area_.height()),
    231                     false);
    232   if (buffer_data.is_null()) {
    233     LOG(WARNING) << "Not enough memory for frame buffers.";
    234     return NULL;
    235   }
    236 
    237   webrtc::DesktopFrame* buffer = new PepperDesktopFrame(buffer_data);
    238   buffers_.push_back(buffer);
    239   return buffer;
    240 }
    241 
    242 void PepperView::FreeBuffer(webrtc::DesktopFrame* buffer) {
    243   DCHECK(std::find(buffers_.begin(), buffers_.end(), buffer) != buffers_.end());
    244 
    245   buffers_.remove(buffer);
    246   delete buffer;
    247 }
    248 
    249 void PepperView::FlushBuffer(const webrtc::DesktopRect& clip_area,
    250                              webrtc::DesktopFrame* buffer,
    251                              const webrtc::DesktopRegion& region) {
    252   // Defer drawing if the flush is already in progress.
    253   if (flush_pending_) {
    254     // |merge_buffer_| is guaranteed to be free here because we allocate only
    255     // two buffers simultaneously. If more buffers are allowed this code should
    256     // apply all pending changes to the screen.
    257     DCHECK(merge_buffer_ == NULL);
    258 
    259     merge_clip_area_ = clip_area;
    260     merge_buffer_ = buffer;
    261     merge_region_ = region;
    262     return;
    263   }
    264 
    265   // Notify Pepper API about the updated areas and flush pixels to the screen.
    266   base::Time start_time = base::Time::Now();
    267 
    268   for (webrtc::DesktopRegion::Iterator i(region); !i.IsAtEnd(); i.Advance()) {
    269     webrtc::DesktopRect rect = i.rect();
    270 
    271     // Re-clip |region| with the current clipping area |clip_area_| because
    272     // the latter could change from the time the buffer was drawn.
    273     rect.IntersectWith(clip_area_);
    274     if (rect.is_empty())
    275       continue;
    276 
    277     // Specify the rectangle coordinates relative to the clipping area.
    278     rect.Translate(-clip_area.left(), -clip_area.top());
    279 
    280     // Pepper Graphics 2D has a strange and badly documented API that the
    281     // point here is the offset from the source rect. Why?
    282     graphics2d_.PaintImageData(
    283         static_cast<PepperDesktopFrame*>(buffer)->buffer(),
    284         pp::Point(clip_area.left(), clip_area.top()),
    285         pp::Rect(rect.left(), rect.top(), rect.width(), rect.height()));
    286   }
    287 
    288   // Notify the producer that some parts of the region weren't painted because
    289   // the clipping area has changed already.
    290   if (!clip_area.equals(clip_area_)) {
    291     webrtc::DesktopRegion not_painted = region;
    292     not_painted.Subtract(clip_area_);
    293     if (!not_painted.is_empty()) {
    294       producer_->InvalidateRegion(not_painted);
    295     }
    296   }
    297 
    298   // Flush the updated areas to the screen.
    299   pp::CompletionCallback callback =
    300       callback_factory_.NewCallback(&PepperView::OnFlushDone,
    301                                     start_time,
    302                                     buffer);
    303   int error = graphics2d_.Flush(callback);
    304   CHECK(error == PP_OK_COMPLETIONPENDING);
    305   flush_pending_ = true;
    306 
    307   // If the buffer we just rendered has a shape then pass that to JavaScript.
    308   const webrtc::DesktopRegion* buffer_shape = producer_->GetBufferShape();
    309   if (buffer_shape)
    310     instance_->SetDesktopShape(*buffer_shape);
    311 }
    312 
    313 void PepperView::OnFlushDone(int result,
    314                              const base::Time& paint_start,
    315                              webrtc::DesktopFrame* buffer) {
    316   DCHECK(context_->main_task_runner()->BelongsToCurrentThread());
    317   DCHECK(flush_pending_);
    318 
    319   instance_->GetStats()->video_paint_ms()->Record(
    320       (base::Time::Now() - paint_start).InMilliseconds());
    321 
    322   flush_pending_ = false;
    323   ReturnBuffer(buffer);
    324 
    325   // If there is a buffer queued for rendering then render it now.
    326   if (merge_buffer_ != NULL) {
    327     buffer = merge_buffer_;
    328     merge_buffer_ = NULL;
    329     FlushBuffer(merge_clip_area_, buffer, merge_region_);
    330   }
    331 }
    332 
    333 }  // namespace remoting
    334