1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "remoting/client/plugin/pepper_view.h" 6 7 #include <functional> 8 9 #include "base/message_loop/message_loop.h" 10 #include "base/strings/string_util.h" 11 #include "base/synchronization/waitable_event.h" 12 #include "base/time/time.h" 13 #include "ppapi/cpp/completion_callback.h" 14 #include "ppapi/cpp/image_data.h" 15 #include "ppapi/cpp/point.h" 16 #include "ppapi/cpp/rect.h" 17 #include "ppapi/cpp/size.h" 18 #include "remoting/base/util.h" 19 #include "remoting/client/chromoting_stats.h" 20 #include "remoting/client/client_context.h" 21 #include "remoting/client/frame_producer.h" 22 #include "remoting/client/plugin/chromoting_instance.h" 23 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h" 24 25 using base::Passed; 26 27 namespace { 28 29 // DesktopFrame that wraps a supplied pp::ImageData 30 class PepperDesktopFrame : public webrtc::DesktopFrame { 31 public: 32 // Wraps the supplied ImageData. 33 explicit PepperDesktopFrame(const pp::ImageData& buffer); 34 35 // Access to underlying pepper representation. 36 const pp::ImageData& buffer() const { 37 return buffer_; 38 } 39 40 private: 41 pp::ImageData buffer_; 42 }; 43 44 PepperDesktopFrame::PepperDesktopFrame(const pp::ImageData& buffer) 45 : DesktopFrame(webrtc::DesktopSize(buffer.size().width(), 46 buffer.size().height()), 47 buffer.stride(), 48 reinterpret_cast<uint8_t*>(buffer.data()), 49 NULL), 50 buffer_(buffer) {} 51 52 } // namespace 53 54 namespace remoting { 55 56 namespace { 57 58 // The maximum number of image buffers to be allocated at any point of time. 59 const size_t kMaxPendingBuffersCount = 2; 60 61 } // namespace 62 63 PepperView::PepperView(ChromotingInstance* instance, 64 ClientContext* context) 65 : instance_(instance), 66 context_(context), 67 producer_(NULL), 68 merge_buffer_(NULL), 69 dips_to_device_scale_(1.0f), 70 dips_to_view_scale_(1.0f), 71 flush_pending_(false), 72 is_initialized_(false), 73 frame_received_(false), 74 callback_factory_(this) { 75 } 76 77 PepperView::~PepperView() { 78 // The producer should now return any pending buffers. At this point, however, 79 // ReturnBuffer() tasks scheduled by the producer will not be delivered, 80 // so we free all the buffers once the producer's queue is empty. 81 base::WaitableEvent done_event(true, false); 82 producer_->RequestReturnBuffers( 83 base::Bind(&base::WaitableEvent::Signal, base::Unretained(&done_event))); 84 done_event.Wait(); 85 86 merge_buffer_ = NULL; 87 while (!buffers_.empty()) { 88 FreeBuffer(buffers_.front()); 89 } 90 } 91 92 void PepperView::Initialize(FrameProducer* producer) { 93 producer_ = producer; 94 webrtc::DesktopFrame* buffer = AllocateBuffer(); 95 while (buffer) { 96 producer_->DrawBuffer(buffer); 97 buffer = AllocateBuffer(); 98 } 99 } 100 101 void PepperView::SetView(const pp::View& view) { 102 bool view_changed = false; 103 104 pp::Rect pp_size = view.GetRect(); 105 webrtc::DesktopSize new_dips_size(pp_size.width(), pp_size.height()); 106 float new_dips_to_device_scale = view.GetDeviceScale(); 107 108 if (!dips_size_.equals(new_dips_size) || 109 dips_to_device_scale_ != new_dips_to_device_scale) { 110 view_changed = true; 111 dips_to_device_scale_ = new_dips_to_device_scale; 112 dips_size_ = new_dips_size; 113 114 // If |dips_to_device_scale_| is > 1.0 then the device is high-DPI, and 115 // there are actually |view_device_scale_| physical pixels for every one 116 // Density Independent Pixel (DIP). If we specify a scale of 1.0 to 117 // Graphics2D then we can render at DIP resolution and let PPAPI up-scale 118 // for high-DPI devices. 119 dips_to_view_scale_ = 1.0f; 120 view_size_ = dips_size_; 121 122 // If the view's DIP dimensions don't match the source then let the frame 123 // producer do the scaling, and render at device resolution. 124 if (!dips_size_.equals(source_size_)) { 125 dips_to_view_scale_ = dips_to_device_scale_; 126 view_size_.set(ceilf(dips_size_.width() * dips_to_view_scale_), 127 ceilf(dips_size_.height() * dips_to_view_scale_)); 128 } 129 130 // Create a 2D rendering context at the chosen frame dimensions. 131 pp::Size pp_size = pp::Size(view_size_.width(), view_size_.height()); 132 graphics2d_ = pp::Graphics2D(instance_, pp_size, false); 133 134 // Specify the scale from our coordinates to DIPs. 135 graphics2d_.SetScale(1.0f / dips_to_view_scale_); 136 137 bool result = instance_->BindGraphics(graphics2d_); 138 139 // There is no good way to handle this error currently. 140 DCHECK(result) << "Couldn't bind the device context."; 141 } 142 143 // Ignore clip rectangle provided by the browser because it may not be 144 // correct. See crbug.com/360240 . In case when the plugin is not visible 145 // (e.g. another tab is selected) |clip_area_| is set to empty rectangle, 146 // otherwise it's set to a rectangle that covers the whole plugin. 147 // 148 // TODO(sergeyu): Use view.GetClipRect() here after bug 360240 is fixed. 149 webrtc::DesktopRect new_clip = 150 view.IsVisible() ? webrtc::DesktopRect::MakeWH( 151 ceilf(pp_size.width() * dips_to_view_scale_), 152 ceilf(pp_size.height() * dips_to_view_scale_)) 153 : webrtc::DesktopRect(); 154 if (!clip_area_.equals(new_clip)) { 155 view_changed = true; 156 157 // YUV to RGB conversion may require even X and Y coordinates for 158 // the top left corner of the clipping area. 159 clip_area_ = AlignRect(new_clip); 160 clip_area_.IntersectWith(webrtc::DesktopRect::MakeSize(view_size_)); 161 } 162 163 if (view_changed) { 164 producer_->SetOutputSizeAndClip(view_size_, clip_area_); 165 Initialize(producer_); 166 } 167 } 168 169 void PepperView::ApplyBuffer(const webrtc::DesktopSize& view_size, 170 const webrtc::DesktopRect& clip_area, 171 webrtc::DesktopFrame* buffer, 172 const webrtc::DesktopRegion& region, 173 const webrtc::DesktopRegion& shape) { 174 DCHECK(context_->main_task_runner()->BelongsToCurrentThread()); 175 176 if (!frame_received_) { 177 instance_->OnFirstFrameReceived(); 178 frame_received_ = true; 179 } 180 // We cannot use the data in the buffer if its dimensions don't match the 181 // current view size. 182 // TODO(alexeypa): We could rescale and draw it (or even draw it without 183 // rescaling) to reduce the perceived lag while we are waiting for 184 // the properly scaled data. 185 if (!view_size_.equals(view_size)) { 186 FreeBuffer(buffer); 187 Initialize(producer_); 188 } else { 189 FlushBuffer(clip_area, buffer, region); 190 instance_->SetDesktopShape(shape); 191 } 192 } 193 194 void PepperView::ReturnBuffer(webrtc::DesktopFrame* buffer) { 195 DCHECK(context_->main_task_runner()->BelongsToCurrentThread()); 196 197 // Reuse the buffer if it is large enough, otherwise drop it on the floor 198 // and allocate a new one. 199 if (buffer->size().width() >= clip_area_.width() && 200 buffer->size().height() >= clip_area_.height()) { 201 producer_->DrawBuffer(buffer); 202 } else { 203 FreeBuffer(buffer); 204 Initialize(producer_); 205 } 206 } 207 208 void PepperView::SetSourceSize(const webrtc::DesktopSize& source_size, 209 const webrtc::DesktopVector& source_dpi) { 210 DCHECK(context_->main_task_runner()->BelongsToCurrentThread()); 211 212 if (source_size_.equals(source_size) && source_dpi_.equals(source_dpi)) 213 return; 214 215 source_size_ = source_size; 216 source_dpi_ = source_dpi; 217 218 // Notify JavaScript of the change in source size. 219 instance_->SetDesktopSize(source_size, source_dpi); 220 } 221 222 FrameConsumer::PixelFormat PepperView::GetPixelFormat() { 223 return FORMAT_BGRA; 224 } 225 226 webrtc::DesktopFrame* PepperView::AllocateBuffer() { 227 if (buffers_.size() >= kMaxPendingBuffersCount) 228 return NULL; 229 230 if (clip_area_.width()==0 || clip_area_.height()==0) 231 return NULL; 232 233 // Create an image buffer of the required size, but don't zero it. 234 pp::ImageData buffer_data(instance_, 235 PP_IMAGEDATAFORMAT_BGRA_PREMUL, 236 pp::Size(clip_area_.width(), 237 clip_area_.height()), 238 false); 239 if (buffer_data.is_null()) { 240 LOG(WARNING) << "Not enough memory for frame buffers."; 241 return NULL; 242 } 243 244 webrtc::DesktopFrame* buffer = new PepperDesktopFrame(buffer_data); 245 buffers_.push_back(buffer); 246 return buffer; 247 } 248 249 void PepperView::FreeBuffer(webrtc::DesktopFrame* buffer) { 250 DCHECK(std::find(buffers_.begin(), buffers_.end(), buffer) != buffers_.end()); 251 252 buffers_.remove(buffer); 253 delete buffer; 254 } 255 256 void PepperView::FlushBuffer(const webrtc::DesktopRect& clip_area, 257 webrtc::DesktopFrame* buffer, 258 const webrtc::DesktopRegion& region) { 259 // Defer drawing if the flush is already in progress. 260 if (flush_pending_) { 261 // |merge_buffer_| is guaranteed to be free here because we allocate only 262 // two buffers simultaneously. If more buffers are allowed this code should 263 // apply all pending changes to the screen. 264 DCHECK(merge_buffer_ == NULL); 265 266 merge_clip_area_ = clip_area; 267 merge_buffer_ = buffer; 268 merge_region_ = region; 269 return; 270 } 271 272 // Notify Pepper API about the updated areas and flush pixels to the screen. 273 base::Time start_time = base::Time::Now(); 274 275 for (webrtc::DesktopRegion::Iterator i(region); !i.IsAtEnd(); i.Advance()) { 276 webrtc::DesktopRect rect = i.rect(); 277 278 // Re-clip |region| with the current clipping area |clip_area_| because 279 // the latter could change from the time the buffer was drawn. 280 rect.IntersectWith(clip_area_); 281 if (rect.is_empty()) 282 continue; 283 284 // Specify the rectangle coordinates relative to the clipping area. 285 rect.Translate(-clip_area.left(), -clip_area.top()); 286 287 // Pepper Graphics 2D has a strange and badly documented API that the 288 // point here is the offset from the source rect. Why? 289 graphics2d_.PaintImageData( 290 static_cast<PepperDesktopFrame*>(buffer)->buffer(), 291 pp::Point(clip_area.left(), clip_area.top()), 292 pp::Rect(rect.left(), rect.top(), rect.width(), rect.height())); 293 } 294 295 // Notify the producer that some parts of the region weren't painted because 296 // the clipping area has changed already. 297 if (!clip_area.equals(clip_area_)) { 298 webrtc::DesktopRegion not_painted = region; 299 not_painted.Subtract(clip_area_); 300 if (!not_painted.is_empty()) { 301 producer_->InvalidateRegion(not_painted); 302 } 303 } 304 305 // Flush the updated areas to the screen. 306 pp::CompletionCallback callback = 307 callback_factory_.NewCallback(&PepperView::OnFlushDone, 308 start_time, 309 buffer); 310 int error = graphics2d_.Flush(callback); 311 CHECK(error == PP_OK_COMPLETIONPENDING); 312 flush_pending_ = true; 313 } 314 315 void PepperView::OnFlushDone(int result, 316 const base::Time& paint_start, 317 webrtc::DesktopFrame* buffer) { 318 DCHECK(context_->main_task_runner()->BelongsToCurrentThread()); 319 DCHECK(flush_pending_); 320 321 instance_->GetStats()->video_paint_ms()->Record( 322 (base::Time::Now() - paint_start).InMilliseconds()); 323 324 flush_pending_ = false; 325 ReturnBuffer(buffer); 326 327 // If there is a buffer queued for rendering then render it now. 328 if (merge_buffer_ != NULL) { 329 buffer = merge_buffer_; 330 merge_buffer_ = NULL; 331 FlushBuffer(merge_clip_area_, buffer, merge_region_); 332 } 333 } 334 335 } // namespace remoting 336
