1 // Copyright 2014 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "remoting/client/software_video_renderer.h" 6 7 #include <list> 8 9 #include "base/bind.h" 10 #include "base/callback.h" 11 #include "base/callback_helpers.h" 12 #include "base/location.h" 13 #include "base/logging.h" 14 #include "base/single_thread_task_runner.h" 15 #include "remoting/base/util.h" 16 #include "remoting/client/frame_consumer.h" 17 #include "remoting/codec/video_decoder.h" 18 #include "remoting/codec/video_decoder_verbatim.h" 19 #if !defined(MEDIA_DISABLE_LIBVPX) 20 #include "remoting/codec/video_decoder_vpx.h" 21 #endif // !defined(MEDIA_DISABLE_LIBVPX) 22 #include "remoting/protocol/session_config.h" 23 #include "third_party/libyuv/include/libyuv/convert_argb.h" 24 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h" 25 26 using base::Passed; 27 using remoting::protocol::ChannelConfig; 28 using remoting::protocol::SessionConfig; 29 30 namespace remoting { 31 32 // This class wraps a VideoDecoder and byte-swaps the pixels for compatibility 33 // with the android.graphics.Bitmap class. 34 // TODO(lambroslambrou): Refactor so that the VideoDecoder produces data 35 // in the right byte-order, instead of swapping it here. 36 class RgbToBgrVideoDecoderFilter : public VideoDecoder { 37 public: 38 RgbToBgrVideoDecoderFilter(scoped_ptr<VideoDecoder> parent) 39 : parent_(parent.Pass()) { 40 } 41 42 virtual void Initialize(const webrtc::DesktopSize& screen_size) OVERRIDE { 43 parent_->Initialize(screen_size); 44 } 45 46 virtual bool DecodePacket(const VideoPacket& packet) OVERRIDE { 47 return parent_->DecodePacket(packet); 48 } 49 50 virtual void Invalidate(const webrtc::DesktopSize& view_size, 51 const webrtc::DesktopRegion& region) OVERRIDE { 52 return parent_->Invalidate(view_size, region); 53 } 54 55 virtual void RenderFrame(const webrtc::DesktopSize& view_size, 56 const webrtc::DesktopRect& clip_area, 57 uint8* image_buffer, 58 int image_stride, 59 webrtc::DesktopRegion* output_region) OVERRIDE { 60 parent_->RenderFrame(view_size, clip_area, image_buffer, image_stride, 61 output_region); 62 63 for (webrtc::DesktopRegion::Iterator i(*output_region); !i.IsAtEnd(); 64 i.Advance()) { 65 webrtc::DesktopRect rect = i.rect(); 66 uint8* pixels = image_buffer + (rect.top() * image_stride) + 67 (rect.left() * kBytesPerPixel); 68 libyuv::ABGRToARGB(pixels, image_stride, pixels, image_stride, 69 rect.width(), rect.height()); 70 } 71 } 72 73 virtual const webrtc::DesktopRegion* GetImageShape() OVERRIDE { 74 return parent_->GetImageShape(); 75 } 76 77 private: 78 scoped_ptr<VideoDecoder> parent_; 79 }; 80 81 class SoftwareVideoRenderer::Core { 82 public: 83 Core(scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, 84 scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, 85 scoped_refptr<FrameConsumerProxy> consumer); 86 ~Core(); 87 88 void Initialize(const protocol::SessionConfig& config); 89 void DrawBuffer(webrtc::DesktopFrame* buffer); 90 void InvalidateRegion(const webrtc::DesktopRegion& region); 91 void RequestReturnBuffers(const base::Closure& done); 92 void SetOutputSizeAndClip( 93 const webrtc::DesktopSize& view_size, 94 const webrtc::DesktopRect& clip_area); 95 96 // Decodes the contents of |packet|. DecodePacket may keep a reference to 97 // |packet| so the |packet| must remain alive and valid until |done| is 98 // executed. 99 void DecodePacket(scoped_ptr<VideoPacket> packet, const base::Closure& done); 100 101 private: 102 // Paints the invalidated region to the next available buffer and returns it 103 // to the consumer. 104 void SchedulePaint(); 105 void DoPaint(); 106 107 scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_; 108 scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner_; 109 scoped_refptr<FrameConsumerProxy> consumer_; 110 scoped_ptr<VideoDecoder> decoder_; 111 112 // Remote screen size in pixels. 113 webrtc::DesktopSize source_size_; 114 115 // Vertical and horizontal DPI of the remote screen. 116 webrtc::DesktopVector source_dpi_; 117 118 // The current dimensions of the frame consumer view. 119 webrtc::DesktopSize view_size_; 120 webrtc::DesktopRect clip_area_; 121 122 // The drawing buffers supplied by the frame consumer. 123 std::list<webrtc::DesktopFrame*> buffers_; 124 125 // Flag used to coalesce runs of SchedulePaint()s into a single DoPaint(). 126 bool paint_scheduled_; 127 128 base::WeakPtrFactory<Core> weak_factory_; 129 }; 130 131 SoftwareVideoRenderer::Core::Core( 132 scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, 133 scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, 134 scoped_refptr<FrameConsumerProxy> consumer) 135 : main_task_runner_(main_task_runner), 136 decode_task_runner_(decode_task_runner), 137 consumer_(consumer), 138 paint_scheduled_(false), 139 weak_factory_(this) { 140 } 141 142 SoftwareVideoRenderer::Core::~Core() { 143 } 144 145 void SoftwareVideoRenderer::Core::Initialize(const SessionConfig& config) { 146 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 147 148 // Initialize decoder based on the selected codec. 149 ChannelConfig::Codec codec = config.video_config().codec; 150 if (codec == ChannelConfig::CODEC_VERBATIM) { 151 decoder_.reset(new VideoDecoderVerbatim()); 152 #if !defined(MEDIA_DISABLE_LIBVPX) 153 } else if (codec == ChannelConfig::CODEC_VP8) { 154 decoder_ = VideoDecoderVpx::CreateForVP8(); 155 } else if (codec == ChannelConfig::CODEC_VP9) { 156 decoder_ = VideoDecoderVpx::CreateForVP9(); 157 } else { 158 #endif // !defined(MEDIA_DISABLE_LIBVPX) 159 NOTREACHED() << "Invalid Encoding found: " << codec; 160 } 161 162 if (consumer_->GetPixelFormat() == FrameConsumer::FORMAT_RGBA) { 163 scoped_ptr<VideoDecoder> wrapper( 164 new RgbToBgrVideoDecoderFilter(decoder_.Pass())); 165 decoder_ = wrapper.Pass(); 166 } 167 } 168 169 void SoftwareVideoRenderer::Core::DecodePacket(scoped_ptr<VideoPacket> packet, 170 const base::Closure& done) { 171 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 172 173 bool decoder_needs_reset = false; 174 bool notify_size_or_dpi_change = false; 175 176 // If the packet includes screen size or DPI information, store them. 177 if (packet->format().has_screen_width() && 178 packet->format().has_screen_height()) { 179 webrtc::DesktopSize source_size(packet->format().screen_width(), 180 packet->format().screen_height()); 181 if (!source_size_.equals(source_size)) { 182 source_size_ = source_size; 183 decoder_needs_reset = true; 184 notify_size_or_dpi_change = true; 185 } 186 } 187 if (packet->format().has_x_dpi() && packet->format().has_y_dpi()) { 188 webrtc::DesktopVector source_dpi(packet->format().x_dpi(), 189 packet->format().y_dpi()); 190 if (!source_dpi.equals(source_dpi_)) { 191 source_dpi_ = source_dpi; 192 notify_size_or_dpi_change = true; 193 } 194 } 195 196 // If we've never seen a screen size, ignore the packet. 197 if (source_size_.is_empty()) { 198 main_task_runner_->PostTask(FROM_HERE, base::Bind(done)); 199 return; 200 } 201 202 if (decoder_needs_reset) 203 decoder_->Initialize(source_size_); 204 if (notify_size_or_dpi_change) 205 consumer_->SetSourceSize(source_size_, source_dpi_); 206 207 if (decoder_->DecodePacket(*packet.get())) { 208 SchedulePaint(); 209 } else { 210 LOG(ERROR) << "DecodePacket() failed."; 211 } 212 213 main_task_runner_->PostTask(FROM_HERE, base::Bind(done)); 214 } 215 216 void SoftwareVideoRenderer::Core::SchedulePaint() { 217 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 218 if (paint_scheduled_) 219 return; 220 paint_scheduled_ = true; 221 decode_task_runner_->PostTask( 222 FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::DoPaint, 223 weak_factory_.GetWeakPtr())); 224 } 225 226 void SoftwareVideoRenderer::Core::DoPaint() { 227 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 228 DCHECK(paint_scheduled_); 229 paint_scheduled_ = false; 230 231 // If the view size is empty or we have no output buffers ready, return. 232 if (buffers_.empty() || view_size_.is_empty()) 233 return; 234 235 // If no Decoder is initialized, or the host dimensions are empty, return. 236 if (!decoder_.get() || source_size_.is_empty()) 237 return; 238 239 // Draw the invalidated region to the buffer. 240 webrtc::DesktopFrame* buffer = buffers_.front(); 241 webrtc::DesktopRegion output_region; 242 decoder_->RenderFrame(view_size_, clip_area_, 243 buffer->data(), buffer->stride(), &output_region); 244 245 // Notify the consumer that painting is done. 246 if (!output_region.is_empty()) { 247 buffers_.pop_front(); 248 consumer_->ApplyBuffer(view_size_, clip_area_, buffer, output_region, 249 *decoder_->GetImageShape()); 250 } 251 } 252 253 void SoftwareVideoRenderer::Core::RequestReturnBuffers( 254 const base::Closure& done) { 255 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 256 257 while (!buffers_.empty()) { 258 consumer_->ReturnBuffer(buffers_.front()); 259 buffers_.pop_front(); 260 } 261 262 if (!done.is_null()) 263 done.Run(); 264 } 265 266 void SoftwareVideoRenderer::Core::DrawBuffer(webrtc::DesktopFrame* buffer) { 267 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 268 DCHECK(clip_area_.width() <= buffer->size().width() && 269 clip_area_.height() <= buffer->size().height()); 270 271 buffers_.push_back(buffer); 272 SchedulePaint(); 273 } 274 275 void SoftwareVideoRenderer::Core::InvalidateRegion( 276 const webrtc::DesktopRegion& region) { 277 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 278 279 if (decoder_.get()) { 280 decoder_->Invalidate(view_size_, region); 281 SchedulePaint(); 282 } 283 } 284 285 void SoftwareVideoRenderer::Core::SetOutputSizeAndClip( 286 const webrtc::DesktopSize& view_size, 287 const webrtc::DesktopRect& clip_area) { 288 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 289 290 // The whole frame needs to be repainted if the scaling factor has changed. 291 if (!view_size_.equals(view_size) && decoder_.get()) { 292 webrtc::DesktopRegion region; 293 region.AddRect(webrtc::DesktopRect::MakeSize(view_size)); 294 decoder_->Invalidate(view_size, region); 295 } 296 297 if (!view_size_.equals(view_size) || 298 !clip_area_.equals(clip_area)) { 299 view_size_ = view_size; 300 clip_area_ = clip_area; 301 302 // Return buffers that are smaller than needed to the consumer for 303 // reuse/reallocation. 304 std::list<webrtc::DesktopFrame*>::iterator i = buffers_.begin(); 305 while (i != buffers_.end()) { 306 if ((*i)->size().width() < clip_area_.width() || 307 (*i)->size().height() < clip_area_.height()) { 308 consumer_->ReturnBuffer(*i); 309 i = buffers_.erase(i); 310 } else { 311 ++i; 312 } 313 } 314 315 SchedulePaint(); 316 } 317 } 318 319 SoftwareVideoRenderer::SoftwareVideoRenderer( 320 scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, 321 scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, 322 scoped_refptr<FrameConsumerProxy> consumer) 323 : decode_task_runner_(decode_task_runner), 324 core_(new Core(main_task_runner, decode_task_runner, consumer)), 325 latest_sequence_number_(0), 326 weak_factory_(this) { 327 DCHECK(CalledOnValidThread()); 328 } 329 330 SoftwareVideoRenderer::~SoftwareVideoRenderer() { 331 DCHECK(CalledOnValidThread()); 332 decode_task_runner_->DeleteSoon(FROM_HERE, core_.release()); 333 } 334 335 void SoftwareVideoRenderer::Initialize( 336 const protocol::SessionConfig& config) { 337 DCHECK(CalledOnValidThread()); 338 decode_task_runner_->PostTask( 339 FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::Initialize, 340 base::Unretained(core_.get()), config)); 341 } 342 343 ChromotingStats* SoftwareVideoRenderer::GetStats() { 344 DCHECK(CalledOnValidThread()); 345 return &stats_; 346 } 347 348 void SoftwareVideoRenderer::ProcessVideoPacket(scoped_ptr<VideoPacket> packet, 349 const base::Closure& done) { 350 DCHECK(CalledOnValidThread()); 351 352 // If the video packet is empty then drop it. Empty packets are used to 353 // maintain activity on the network. 354 if (!packet->has_data() || packet->data().size() == 0) { 355 done.Run(); 356 return; 357 } 358 359 // Add one frame to the counter. 360 stats_.video_frame_rate()->Record(1); 361 362 // Record other statistics received from host. 363 stats_.video_bandwidth()->Record(packet->data().size()); 364 if (packet->has_capture_time_ms()) 365 stats_.video_capture_ms()->Record(packet->capture_time_ms()); 366 if (packet->has_encode_time_ms()) 367 stats_.video_encode_ms()->Record(packet->encode_time_ms()); 368 if (packet->has_client_sequence_number() && 369 packet->client_sequence_number() > latest_sequence_number_) { 370 latest_sequence_number_ = packet->client_sequence_number(); 371 base::TimeDelta round_trip_latency = 372 base::Time::Now() - 373 base::Time::FromInternalValue(packet->client_sequence_number()); 374 stats_.round_trip_ms()->Record(round_trip_latency.InMilliseconds()); 375 } 376 377 // Measure the latency between the last packet being received and presented. 378 base::Time decode_start = base::Time::Now(); 379 380 base::Closure decode_done = base::Bind(&SoftwareVideoRenderer::OnPacketDone, 381 weak_factory_.GetWeakPtr(), 382 decode_start, done); 383 384 decode_task_runner_->PostTask(FROM_HERE, base::Bind( 385 &SoftwareVideoRenderer::Core::DecodePacket, 386 base::Unretained(core_.get()), base::Passed(&packet), decode_done)); 387 } 388 389 void SoftwareVideoRenderer::DrawBuffer(webrtc::DesktopFrame* buffer) { 390 decode_task_runner_->PostTask( 391 FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::DrawBuffer, 392 base::Unretained(core_.get()), buffer)); 393 } 394 395 void SoftwareVideoRenderer::InvalidateRegion( 396 const webrtc::DesktopRegion& region) { 397 decode_task_runner_->PostTask( 398 FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::InvalidateRegion, 399 base::Unretained(core_.get()), region)); 400 } 401 402 void SoftwareVideoRenderer::RequestReturnBuffers(const base::Closure& done) { 403 decode_task_runner_->PostTask( 404 FROM_HERE, 405 base::Bind(&SoftwareVideoRenderer::Core::RequestReturnBuffers, 406 base::Unretained(core_.get()), done)); 407 } 408 409 void SoftwareVideoRenderer::SetOutputSizeAndClip( 410 const webrtc::DesktopSize& view_size, 411 const webrtc::DesktopRect& clip_area) { 412 decode_task_runner_->PostTask( 413 FROM_HERE, 414 base::Bind(&SoftwareVideoRenderer::Core::SetOutputSizeAndClip, 415 base::Unretained(core_.get()), view_size, clip_area)); 416 } 417 418 void SoftwareVideoRenderer::OnPacketDone(base::Time decode_start, 419 const base::Closure& done) { 420 DCHECK(CalledOnValidThread()); 421 422 // Record the latency between the packet being received and presented. 423 stats_.video_decode_ms()->Record( 424 (base::Time::Now() - decode_start).InMilliseconds()); 425 426 done.Run(); 427 } 428 429 } // namespace remoting 430
