1 // Copyright 2014 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "remoting/client/software_video_renderer.h" 6 7 #include <list> 8 9 #include "base/bind.h" 10 #include "base/callback.h" 11 #include "base/callback_helpers.h" 12 #include "base/location.h" 13 #include "base/logging.h" 14 #include "base/single_thread_task_runner.h" 15 #include "remoting/base/util.h" 16 #include "remoting/client/frame_consumer.h" 17 #include "remoting/codec/video_decoder.h" 18 #include "remoting/codec/video_decoder_verbatim.h" 19 #include "remoting/codec/video_decoder_vpx.h" 20 #include "remoting/protocol/session_config.h" 21 #include "third_party/libyuv/include/libyuv/convert_argb.h" 22 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h" 23 24 using base::Passed; 25 using remoting::protocol::ChannelConfig; 26 using remoting::protocol::SessionConfig; 27 28 namespace remoting { 29 30 // This class wraps a VideoDecoder and byte-swaps the pixels for compatibility 31 // with the android.graphics.Bitmap class. 32 // TODO(lambroslambrou): Refactor so that the VideoDecoder produces data 33 // in the right byte-order, instead of swapping it here. 34 class RgbToBgrVideoDecoderFilter : public VideoDecoder { 35 public: 36 RgbToBgrVideoDecoderFilter(scoped_ptr<VideoDecoder> parent) 37 : parent_(parent.Pass()) { 38 } 39 40 virtual void Initialize(const webrtc::DesktopSize& screen_size) OVERRIDE { 41 parent_->Initialize(screen_size); 42 } 43 44 virtual bool DecodePacket(const VideoPacket& packet) OVERRIDE { 45 return parent_->DecodePacket(packet); 46 } 47 48 virtual void Invalidate(const webrtc::DesktopSize& view_size, 49 const webrtc::DesktopRegion& region) OVERRIDE { 50 return parent_->Invalidate(view_size, region); 51 } 52 53 virtual void RenderFrame(const webrtc::DesktopSize& view_size, 54 const webrtc::DesktopRect& clip_area, 55 uint8* image_buffer, 56 int image_stride, 57 webrtc::DesktopRegion* output_region) OVERRIDE { 58 parent_->RenderFrame(view_size, clip_area, image_buffer, image_stride, 59 output_region); 60 61 for (webrtc::DesktopRegion::Iterator i(*output_region); !i.IsAtEnd(); 62 i.Advance()) { 63 webrtc::DesktopRect rect = i.rect(); 64 uint8* pixels = image_buffer + (rect.top() * image_stride) + 65 (rect.left() * kBytesPerPixel); 66 libyuv::ABGRToARGB(pixels, image_stride, pixels, image_stride, 67 rect.width(), rect.height()); 68 } 69 } 70 71 virtual const webrtc::DesktopRegion* GetImageShape() OVERRIDE { 72 return parent_->GetImageShape(); 73 } 74 75 private: 76 scoped_ptr<VideoDecoder> parent_; 77 }; 78 79 class SoftwareVideoRenderer::Core { 80 public: 81 Core(scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, 82 scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, 83 scoped_refptr<FrameConsumerProxy> consumer); 84 ~Core(); 85 86 void Initialize(const protocol::SessionConfig& config); 87 void DrawBuffer(webrtc::DesktopFrame* buffer); 88 void InvalidateRegion(const webrtc::DesktopRegion& region); 89 void RequestReturnBuffers(const base::Closure& done); 90 void SetOutputSizeAndClip( 91 const webrtc::DesktopSize& view_size, 92 const webrtc::DesktopRect& clip_area); 93 94 // Decodes the contents of |packet|. DecodePacket may keep a reference to 95 // |packet| so the |packet| must remain alive and valid until |done| is 96 // executed. 97 void DecodePacket(scoped_ptr<VideoPacket> packet, const base::Closure& done); 98 99 private: 100 // Paints the invalidated region to the next available buffer and returns it 101 // to the consumer. 102 void SchedulePaint(); 103 void DoPaint(); 104 105 scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_; 106 scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner_; 107 scoped_refptr<FrameConsumerProxy> consumer_; 108 scoped_ptr<VideoDecoder> decoder_; 109 110 // Remote screen size in pixels. 111 webrtc::DesktopSize source_size_; 112 113 // Vertical and horizontal DPI of the remote screen. 114 webrtc::DesktopVector source_dpi_; 115 116 // The current dimensions of the frame consumer view. 117 webrtc::DesktopSize view_size_; 118 webrtc::DesktopRect clip_area_; 119 120 // The drawing buffers supplied by the frame consumer. 121 std::list<webrtc::DesktopFrame*> buffers_; 122 123 // Flag used to coalesce runs of SchedulePaint()s into a single DoPaint(). 124 bool paint_scheduled_; 125 126 base::WeakPtrFactory<Core> weak_factory_; 127 }; 128 129 SoftwareVideoRenderer::Core::Core( 130 scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, 131 scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, 132 scoped_refptr<FrameConsumerProxy> consumer) 133 : main_task_runner_(main_task_runner), 134 decode_task_runner_(decode_task_runner), 135 consumer_(consumer), 136 paint_scheduled_(false), 137 weak_factory_(this) { 138 } 139 140 SoftwareVideoRenderer::Core::~Core() { 141 } 142 143 void SoftwareVideoRenderer::Core::Initialize(const SessionConfig& config) { 144 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 145 146 // Initialize decoder based on the selected codec. 147 ChannelConfig::Codec codec = config.video_config().codec; 148 if (codec == ChannelConfig::CODEC_VERBATIM) { 149 decoder_.reset(new VideoDecoderVerbatim()); 150 } else if (codec == ChannelConfig::CODEC_VP8) { 151 decoder_ = VideoDecoderVpx::CreateForVP8(); 152 } else if (codec == ChannelConfig::CODEC_VP9) { 153 decoder_ = VideoDecoderVpx::CreateForVP9(); 154 } else { 155 NOTREACHED() << "Invalid Encoding found: " << codec; 156 } 157 158 if (consumer_->GetPixelFormat() == FrameConsumer::FORMAT_RGBA) { 159 scoped_ptr<VideoDecoder> wrapper( 160 new RgbToBgrVideoDecoderFilter(decoder_.Pass())); 161 decoder_ = wrapper.Pass(); 162 } 163 } 164 165 void SoftwareVideoRenderer::Core::DecodePacket(scoped_ptr<VideoPacket> packet, 166 const base::Closure& done) { 167 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 168 169 bool decoder_needs_reset = false; 170 bool notify_size_or_dpi_change = false; 171 172 // If the packet includes screen size or DPI information, store them. 173 if (packet->format().has_screen_width() && 174 packet->format().has_screen_height()) { 175 webrtc::DesktopSize source_size(packet->format().screen_width(), 176 packet->format().screen_height()); 177 if (!source_size_.equals(source_size)) { 178 source_size_ = source_size; 179 decoder_needs_reset = true; 180 notify_size_or_dpi_change = true; 181 } 182 } 183 if (packet->format().has_x_dpi() && packet->format().has_y_dpi()) { 184 webrtc::DesktopVector source_dpi(packet->format().x_dpi(), 185 packet->format().y_dpi()); 186 if (!source_dpi.equals(source_dpi_)) { 187 source_dpi_ = source_dpi; 188 notify_size_or_dpi_change = true; 189 } 190 } 191 192 // If we've never seen a screen size, ignore the packet. 193 if (source_size_.is_empty()) { 194 main_task_runner_->PostTask(FROM_HERE, base::Bind(done)); 195 return; 196 } 197 198 if (decoder_needs_reset) 199 decoder_->Initialize(source_size_); 200 if (notify_size_or_dpi_change) 201 consumer_->SetSourceSize(source_size_, source_dpi_); 202 203 if (decoder_->DecodePacket(*packet.get())) { 204 SchedulePaint(); 205 } else { 206 LOG(ERROR) << "DecodePacket() failed."; 207 } 208 209 main_task_runner_->PostTask(FROM_HERE, base::Bind(done)); 210 } 211 212 void SoftwareVideoRenderer::Core::SchedulePaint() { 213 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 214 if (paint_scheduled_) 215 return; 216 paint_scheduled_ = true; 217 decode_task_runner_->PostTask( 218 FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::DoPaint, 219 weak_factory_.GetWeakPtr())); 220 } 221 222 void SoftwareVideoRenderer::Core::DoPaint() { 223 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 224 DCHECK(paint_scheduled_); 225 paint_scheduled_ = false; 226 227 // If the view size is empty or we have no output buffers ready, return. 228 if (buffers_.empty() || view_size_.is_empty()) 229 return; 230 231 // If no Decoder is initialized, or the host dimensions are empty, return. 232 if (!decoder_.get() || source_size_.is_empty()) 233 return; 234 235 // Draw the invalidated region to the buffer. 236 webrtc::DesktopFrame* buffer = buffers_.front(); 237 webrtc::DesktopRegion output_region; 238 decoder_->RenderFrame(view_size_, clip_area_, 239 buffer->data(), buffer->stride(), &output_region); 240 241 // Notify the consumer that painting is done. 242 if (!output_region.is_empty()) { 243 buffers_.pop_front(); 244 consumer_->ApplyBuffer(view_size_, clip_area_, buffer, output_region, 245 *decoder_->GetImageShape()); 246 } 247 } 248 249 void SoftwareVideoRenderer::Core::RequestReturnBuffers( 250 const base::Closure& done) { 251 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 252 253 while (!buffers_.empty()) { 254 consumer_->ReturnBuffer(buffers_.front()); 255 buffers_.pop_front(); 256 } 257 258 if (!done.is_null()) 259 done.Run(); 260 } 261 262 void SoftwareVideoRenderer::Core::DrawBuffer(webrtc::DesktopFrame* buffer) { 263 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 264 DCHECK(clip_area_.width() <= buffer->size().width() && 265 clip_area_.height() <= buffer->size().height()); 266 267 buffers_.push_back(buffer); 268 SchedulePaint(); 269 } 270 271 void SoftwareVideoRenderer::Core::InvalidateRegion( 272 const webrtc::DesktopRegion& region) { 273 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 274 275 if (decoder_.get()) { 276 decoder_->Invalidate(view_size_, region); 277 SchedulePaint(); 278 } 279 } 280 281 void SoftwareVideoRenderer::Core::SetOutputSizeAndClip( 282 const webrtc::DesktopSize& view_size, 283 const webrtc::DesktopRect& clip_area) { 284 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 285 286 // The whole frame needs to be repainted if the scaling factor has changed. 287 if (!view_size_.equals(view_size) && decoder_.get()) { 288 webrtc::DesktopRegion region; 289 region.AddRect(webrtc::DesktopRect::MakeSize(view_size)); 290 decoder_->Invalidate(view_size, region); 291 } 292 293 if (!view_size_.equals(view_size) || 294 !clip_area_.equals(clip_area)) { 295 view_size_ = view_size; 296 clip_area_ = clip_area; 297 298 // Return buffers that are smaller than needed to the consumer for 299 // reuse/reallocation. 300 std::list<webrtc::DesktopFrame*>::iterator i = buffers_.begin(); 301 while (i != buffers_.end()) { 302 if ((*i)->size().width() < clip_area_.width() || 303 (*i)->size().height() < clip_area_.height()) { 304 consumer_->ReturnBuffer(*i); 305 i = buffers_.erase(i); 306 } else { 307 ++i; 308 } 309 } 310 311 SchedulePaint(); 312 } 313 } 314 315 SoftwareVideoRenderer::SoftwareVideoRenderer( 316 scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, 317 scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, 318 scoped_refptr<FrameConsumerProxy> consumer) 319 : decode_task_runner_(decode_task_runner), 320 core_(new Core(main_task_runner, decode_task_runner, consumer)), 321 latest_sequence_number_(0), 322 weak_factory_(this) { 323 DCHECK(CalledOnValidThread()); 324 } 325 326 SoftwareVideoRenderer::~SoftwareVideoRenderer() { 327 DCHECK(CalledOnValidThread()); 328 decode_task_runner_->DeleteSoon(FROM_HERE, core_.release()); 329 } 330 331 void SoftwareVideoRenderer::Initialize( 332 const protocol::SessionConfig& config) { 333 DCHECK(CalledOnValidThread()); 334 decode_task_runner_->PostTask( 335 FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::Initialize, 336 base::Unretained(core_.get()), config)); 337 } 338 339 ChromotingStats* SoftwareVideoRenderer::GetStats() { 340 DCHECK(CalledOnValidThread()); 341 return &stats_; 342 } 343 344 void SoftwareVideoRenderer::ProcessVideoPacket(scoped_ptr<VideoPacket> packet, 345 const base::Closure& done) { 346 DCHECK(CalledOnValidThread()); 347 348 // If the video packet is empty then drop it. Empty packets are used to 349 // maintain activity on the network. 350 if (!packet->has_data() || packet->data().size() == 0) { 351 done.Run(); 352 return; 353 } 354 355 // Add one frame to the counter. 356 stats_.video_frame_rate()->Record(1); 357 358 // Record other statistics received from host. 359 stats_.video_bandwidth()->Record(packet->data().size()); 360 if (packet->has_capture_time_ms()) 361 stats_.video_capture_ms()->Record(packet->capture_time_ms()); 362 if (packet->has_encode_time_ms()) 363 stats_.video_encode_ms()->Record(packet->encode_time_ms()); 364 if (packet->has_client_sequence_number() && 365 packet->client_sequence_number() > latest_sequence_number_) { 366 latest_sequence_number_ = packet->client_sequence_number(); 367 base::TimeDelta round_trip_latency = 368 base::Time::Now() - 369 base::Time::FromInternalValue(packet->client_sequence_number()); 370 stats_.round_trip_ms()->Record(round_trip_latency.InMilliseconds()); 371 } 372 373 // Measure the latency between the last packet being received and presented. 374 base::Time decode_start = base::Time::Now(); 375 376 base::Closure decode_done = base::Bind(&SoftwareVideoRenderer::OnPacketDone, 377 weak_factory_.GetWeakPtr(), 378 decode_start, done); 379 380 decode_task_runner_->PostTask(FROM_HERE, base::Bind( 381 &SoftwareVideoRenderer::Core::DecodePacket, 382 base::Unretained(core_.get()), base::Passed(&packet), decode_done)); 383 } 384 385 void SoftwareVideoRenderer::DrawBuffer(webrtc::DesktopFrame* buffer) { 386 decode_task_runner_->PostTask( 387 FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::DrawBuffer, 388 base::Unretained(core_.get()), buffer)); 389 } 390 391 void SoftwareVideoRenderer::InvalidateRegion( 392 const webrtc::DesktopRegion& region) { 393 decode_task_runner_->PostTask( 394 FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::InvalidateRegion, 395 base::Unretained(core_.get()), region)); 396 } 397 398 void SoftwareVideoRenderer::RequestReturnBuffers(const base::Closure& done) { 399 decode_task_runner_->PostTask( 400 FROM_HERE, 401 base::Bind(&SoftwareVideoRenderer::Core::RequestReturnBuffers, 402 base::Unretained(core_.get()), done)); 403 } 404 405 void SoftwareVideoRenderer::SetOutputSizeAndClip( 406 const webrtc::DesktopSize& view_size, 407 const webrtc::DesktopRect& clip_area) { 408 decode_task_runner_->PostTask( 409 FROM_HERE, 410 base::Bind(&SoftwareVideoRenderer::Core::SetOutputSizeAndClip, 411 base::Unretained(core_.get()), view_size, clip_area)); 412 } 413 414 void SoftwareVideoRenderer::OnPacketDone(base::Time decode_start, 415 const base::Closure& done) { 416 DCHECK(CalledOnValidThread()); 417 418 // Record the latency between the packet being received and presented. 419 stats_.video_decode_ms()->Record( 420 (base::Time::Now() - decode_start).InMilliseconds()); 421 422 done.Run(); 423 } 424 425 } // namespace remoting 426