1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "remoting/client/rectangle_update_decoder.h" 6 7 #include "base/bind.h" 8 #include "base/bind_helpers.h" 9 #include "base/callback.h" 10 #include "base/location.h" 11 #include "base/logging.h" 12 #include "base/single_thread_task_runner.h" 13 #include "remoting/base/util.h" 14 #include "remoting/codec/video_decoder.h" 15 #include "remoting/codec/video_decoder_verbatim.h" 16 #include "remoting/codec/video_decoder_vp8.h" 17 #include "remoting/client/frame_consumer.h" 18 #include "remoting/protocol/session_config.h" 19 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h" 20 21 using base::Passed; 22 using remoting::protocol::ChannelConfig; 23 using remoting::protocol::SessionConfig; 24 25 namespace remoting { 26 27 RectangleUpdateDecoder::RectangleUpdateDecoder( 28 scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, 29 scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, 30 scoped_refptr<FrameConsumerProxy> consumer) 31 : main_task_runner_(main_task_runner), 32 decode_task_runner_(decode_task_runner), 33 consumer_(consumer), 34 source_size_(SkISize::Make(0, 0)), 35 source_dpi_(SkIPoint::Make(0, 0)), 36 view_size_(SkISize::Make(0, 0)), 37 clip_area_(SkIRect::MakeEmpty()), 38 paint_scheduled_(false), 39 latest_sequence_number_(0) { 40 } 41 42 RectangleUpdateDecoder::~RectangleUpdateDecoder() { 43 } 44 45 void RectangleUpdateDecoder::Initialize(const SessionConfig& config) { 46 // Initialize decoder based on the selected codec. 47 ChannelConfig::Codec codec = config.video_config().codec; 48 if (codec == ChannelConfig::CODEC_VERBATIM) { 49 decoder_.reset(new VideoDecoderVerbatim()); 50 } else if (codec == ChannelConfig::CODEC_VP8) { 51 decoder_.reset(new VideoDecoderVp8()); 52 } else { 53 NOTREACHED() << "Invalid Encoding found: " << codec; 54 } 55 } 56 57 void RectangleUpdateDecoder::DecodePacket(scoped_ptr<VideoPacket> packet, 58 const base::Closure& done) { 59 DCHECK(decode_task_runner_->BelongsToCurrentThread()); 60 61 base::ScopedClosureRunner done_runner(done); 62 63 bool decoder_needs_reset = false; 64 bool notify_size_or_dpi_change = false; 65 66 // If the packet includes screen size or DPI information, store them. 67 if (packet->format().has_screen_width() && 68 packet->format().has_screen_height()) { 69 SkISize source_size = SkISize::Make(packet->format().screen_width(), 70 packet->format().screen_height()); 71 if (source_size_ != source_size) { 72 source_size_ = source_size; 73 decoder_needs_reset = true; 74 notify_size_or_dpi_change = true; 75 } 76 } 77 if (packet->format().has_x_dpi() && packet->format().has_y_dpi()) { 78 SkIPoint source_dpi(SkIPoint::Make(packet->format().x_dpi(), 79 packet->format().y_dpi())); 80 if (source_dpi != source_dpi_) { 81 source_dpi_ = source_dpi; 82 notify_size_or_dpi_change = true; 83 } 84 } 85 86 // If we've never seen a screen size, ignore the packet. 87 if (source_size_.isZero()) 88 return; 89 90 if (decoder_needs_reset) 91 decoder_->Initialize(source_size_); 92 if (notify_size_or_dpi_change) 93 consumer_->SetSourceSize(source_size_, source_dpi_); 94 95 if (!decoder_->IsReadyForData()) { 96 // TODO(ajwong): This whole thing should move into an invalid state. 97 LOG(ERROR) << "Decoder is unable to process data. Dropping packet."; 98 return; 99 } 100 101 if (decoder_->DecodePacket(packet.get()) == VideoDecoder::DECODE_DONE) 102 SchedulePaint(); 103 } 104 105 void RectangleUpdateDecoder::SchedulePaint() { 106 if (paint_scheduled_) 107 return; 108 paint_scheduled_ = true; 109 decode_task_runner_->PostTask( 110 FROM_HERE, base::Bind(&RectangleUpdateDecoder::DoPaint, this)); 111 } 112 113 void RectangleUpdateDecoder::DoPaint() { 114 DCHECK(paint_scheduled_); 115 paint_scheduled_ = false; 116 117 // If the view size is empty or we have no output buffers ready, return. 118 if (buffers_.empty() || view_size_.isEmpty()) 119 return; 120 121 // If no Decoder is initialized, or the host dimensions are empty, return. 122 if (!decoder_.get() || source_size_.isEmpty()) 123 return; 124 125 // Draw the invalidated region to the buffer. 126 webrtc::DesktopFrame* buffer = buffers_.front(); 127 SkRegion output_region; 128 decoder_->RenderFrame(view_size_, clip_area_, 129 buffer->data(), 130 buffer->stride(), 131 &output_region); 132 133 // Notify the consumer that painting is done. 134 if (!output_region.isEmpty()) { 135 buffers_.pop_front(); 136 consumer_->ApplyBuffer(view_size_, clip_area_, buffer, output_region); 137 } 138 } 139 140 void RectangleUpdateDecoder::RequestReturnBuffers(const base::Closure& done) { 141 if (!decode_task_runner_->BelongsToCurrentThread()) { 142 decode_task_runner_->PostTask( 143 FROM_HERE, base::Bind(&RectangleUpdateDecoder::RequestReturnBuffers, 144 this, done)); 145 return; 146 } 147 148 while (!buffers_.empty()) { 149 consumer_->ReturnBuffer(buffers_.front()); 150 buffers_.pop_front(); 151 } 152 153 if (!done.is_null()) 154 done.Run(); 155 } 156 157 void RectangleUpdateDecoder::DrawBuffer(webrtc::DesktopFrame* buffer) { 158 if (!decode_task_runner_->BelongsToCurrentThread()) { 159 decode_task_runner_->PostTask( 160 FROM_HERE, base::Bind(&RectangleUpdateDecoder::DrawBuffer, 161 this, buffer)); 162 return; 163 } 164 165 DCHECK(clip_area_.width() <= buffer->size().width() && 166 clip_area_.height() <= buffer->size().height()); 167 168 buffers_.push_back(buffer); 169 SchedulePaint(); 170 } 171 172 void RectangleUpdateDecoder::InvalidateRegion(const SkRegion& region) { 173 if (!decode_task_runner_->BelongsToCurrentThread()) { 174 decode_task_runner_->PostTask( 175 FROM_HERE, base::Bind(&RectangleUpdateDecoder::InvalidateRegion, 176 this, region)); 177 return; 178 } 179 180 if (decoder_.get()) { 181 decoder_->Invalidate(view_size_, region); 182 SchedulePaint(); 183 } 184 } 185 186 void RectangleUpdateDecoder::SetOutputSizeAndClip(const SkISize& view_size, 187 const SkIRect& clip_area) { 188 if (!decode_task_runner_->BelongsToCurrentThread()) { 189 decode_task_runner_->PostTask( 190 FROM_HERE, base::Bind(&RectangleUpdateDecoder::SetOutputSizeAndClip, 191 this, view_size, clip_area)); 192 return; 193 } 194 195 // The whole frame needs to be repainted if the scaling factor has changed. 196 if (view_size_ != view_size && decoder_.get()) { 197 SkRegion region; 198 region.op(SkIRect::MakeSize(view_size), SkRegion::kUnion_Op); 199 decoder_->Invalidate(view_size, region); 200 } 201 202 if (view_size_ != view_size || 203 clip_area_ != clip_area) { 204 view_size_ = view_size; 205 clip_area_ = clip_area; 206 207 // Return buffers that are smaller than needed to the consumer for 208 // reuse/reallocation. 209 std::list<webrtc::DesktopFrame*>::iterator i = buffers_.begin(); 210 while (i != buffers_.end()) { 211 if ((*i)->size().width() < clip_area_.width() || 212 (*i)->size().height() < clip_area_.height()) { 213 consumer_->ReturnBuffer(*i); 214 i = buffers_.erase(i); 215 } else { 216 ++i; 217 } 218 } 219 220 SchedulePaint(); 221 } 222 } 223 224 const SkRegion* RectangleUpdateDecoder::GetBufferShape() { 225 return decoder_->GetImageShape(); 226 } 227 228 void RectangleUpdateDecoder::ProcessVideoPacket(scoped_ptr<VideoPacket> packet, 229 const base::Closure& done) { 230 DCHECK(main_task_runner_->BelongsToCurrentThread()); 231 232 // If the video packet is empty then drop it. Empty packets are used to 233 // maintain activity on the network. 234 if (!packet->has_data() || packet->data().size() == 0) { 235 done.Run(); 236 return; 237 } 238 239 // Add one frame to the counter. 240 stats_.video_frame_rate()->Record(1); 241 242 // Record other statistics received from host. 243 stats_.video_bandwidth()->Record(packet->data().size()); 244 if (packet->has_capture_time_ms()) 245 stats_.video_capture_ms()->Record(packet->capture_time_ms()); 246 if (packet->has_encode_time_ms()) 247 stats_.video_encode_ms()->Record(packet->encode_time_ms()); 248 if (packet->has_client_sequence_number() && 249 packet->client_sequence_number() > latest_sequence_number_) { 250 latest_sequence_number_ = packet->client_sequence_number(); 251 base::TimeDelta round_trip_latency = 252 base::Time::Now() - 253 base::Time::FromInternalValue(packet->client_sequence_number()); 254 stats_.round_trip_ms()->Record(round_trip_latency.InMilliseconds()); 255 } 256 257 // Measure the latency between the last packet being received and presented. 258 bool last_packet = (packet->flags() & VideoPacket::LAST_PACKET) != 0; 259 base::Time decode_start; 260 if (last_packet) 261 decode_start = base::Time::Now(); 262 263 base::Closure decode_done = 264 base::Bind(&RectangleUpdateDecoder::OnPacketDone, this, 265 last_packet, decode_start, done); 266 267 decode_task_runner_->PostTask(FROM_HERE, base::Bind( 268 &RectangleUpdateDecoder::DecodePacket, this, 269 base::Passed(&packet), decode_done)); 270 } 271 272 void RectangleUpdateDecoder::OnPacketDone(bool last_packet, 273 base::Time decode_start, 274 const base::Closure& done) { 275 if (!main_task_runner_->BelongsToCurrentThread()) { 276 main_task_runner_->PostTask(FROM_HERE, base::Bind( 277 &RectangleUpdateDecoder::OnPacketDone, this, 278 last_packet, decode_start, done)); 279 return; 280 } 281 282 // Record the latency between the final packet being received and 283 // presented. 284 if (last_packet) { 285 stats_.video_decode_ms()->Record( 286 (base::Time::Now() - decode_start).InMilliseconds()); 287 } 288 289 done.Run(); 290 } 291 292 ChromotingStats* RectangleUpdateDecoder::GetStats() { 293 DCHECK(main_task_runner_->BelongsToCurrentThread()); 294 return &stats_; 295 } 296 297 } // namespace remoting 298