1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "remoting/host/client_session.h" 6 7 #include <algorithm> 8 9 #include "base/message_loop/message_loop_proxy.h" 10 #include "remoting/base/capabilities.h" 11 #include "remoting/base/logging.h" 12 #include "remoting/codec/audio_encoder.h" 13 #include "remoting/codec/audio_encoder_opus.h" 14 #include "remoting/codec/audio_encoder_verbatim.h" 15 #include "remoting/codec/video_encoder.h" 16 #include "remoting/codec/video_encoder_verbatim.h" 17 #include "remoting/codec/video_encoder_vpx.h" 18 #include "remoting/host/audio_capturer.h" 19 #include "remoting/host/audio_scheduler.h" 20 #include "remoting/host/desktop_environment.h" 21 #include "remoting/host/host_extension_session.h" 22 #include "remoting/host/input_injector.h" 23 #include "remoting/host/screen_controls.h" 24 #include "remoting/host/screen_resolution.h" 25 #include "remoting/host/video_scheduler.h" 26 #include "remoting/proto/control.pb.h" 27 #include "remoting/proto/event.pb.h" 28 #include "remoting/protocol/client_stub.h" 29 #include "remoting/protocol/clipboard_thread_proxy.h" 30 #include "remoting/protocol/pairing_registry.h" 31 #include "third_party/webrtc/modules/desktop_capture/desktop_capturer.h" 32 33 // Default DPI to assume for old clients that use notifyClientDimensions. 34 const int kDefaultDPI = 96; 35 36 namespace remoting { 37 38 ClientSession::ClientSession( 39 EventHandler* event_handler, 40 scoped_refptr<base::SingleThreadTaskRunner> audio_task_runner, 41 scoped_refptr<base::SingleThreadTaskRunner> input_task_runner, 42 scoped_refptr<base::SingleThreadTaskRunner> video_capture_task_runner, 43 scoped_refptr<base::SingleThreadTaskRunner> video_encode_task_runner, 44 scoped_refptr<base::SingleThreadTaskRunner> network_task_runner, 45 scoped_refptr<base::SingleThreadTaskRunner> ui_task_runner, 46 scoped_ptr<protocol::ConnectionToClient> connection, 47 DesktopEnvironmentFactory* desktop_environment_factory, 48 const base::TimeDelta& max_duration, 49 scoped_refptr<protocol::PairingRegistry> pairing_registry, 50 const std::vector<HostExtension*>& extensions) 51 : event_handler_(event_handler), 52 connection_(connection.Pass()), 53 client_jid_(connection_->session()->jid()), 54 desktop_environment_factory_(desktop_environment_factory), 55 input_tracker_(&host_input_filter_), 56 remote_input_filter_(&input_tracker_), 57 mouse_clamping_filter_(&remote_input_filter_), 58 disable_input_filter_(mouse_clamping_filter_.input_filter()), 59 disable_clipboard_filter_(clipboard_echo_filter_.host_filter()), 60 auth_input_filter_(&disable_input_filter_), 61 auth_clipboard_filter_(&disable_clipboard_filter_), 62 client_clipboard_factory_(clipboard_echo_filter_.client_filter()), 63 max_duration_(max_duration), 64 audio_task_runner_(audio_task_runner), 65 input_task_runner_(input_task_runner), 66 video_capture_task_runner_(video_capture_task_runner), 67 video_encode_task_runner_(video_encode_task_runner), 68 network_task_runner_(network_task_runner), 69 ui_task_runner_(ui_task_runner), 70 pairing_registry_(pairing_registry), 71 pause_video_(false), 72 lossless_video_encode_(false), 73 lossless_video_color_(false), 74 weak_factory_(this) { 75 connection_->SetEventHandler(this); 76 77 // TODO(sergeyu): Currently ConnectionToClient expects stubs to be 78 // set before channels are connected. Make it possible to set stubs 79 // later and set them only when connection is authenticated. 80 connection_->set_clipboard_stub(&auth_clipboard_filter_); 81 connection_->set_host_stub(this); 82 connection_->set_input_stub(&auth_input_filter_); 83 84 // |auth_*_filter_|'s states reflect whether the session is authenticated. 85 auth_input_filter_.set_enabled(false); 86 auth_clipboard_filter_.set_enabled(false); 87 88 // Create a manager for the configured extensions, if any. 89 extension_manager_.reset(new HostExtensionSessionManager(extensions, this)); 90 91 #if defined(OS_WIN) 92 // LocalInputMonitorWin filters out an echo of the injected input before it 93 // reaches |remote_input_filter_|. 94 remote_input_filter_.SetExpectLocalEcho(false); 95 #endif // defined(OS_WIN) 96 } 97 98 ClientSession::~ClientSession() { 99 DCHECK(CalledOnValidThread()); 100 DCHECK(!audio_scheduler_.get()); 101 DCHECK(!desktop_environment_); 102 DCHECK(!input_injector_); 103 DCHECK(!screen_controls_); 104 DCHECK(!video_scheduler_.get()); 105 106 connection_.reset(); 107 } 108 109 void ClientSession::NotifyClientResolution( 110 const protocol::ClientResolution& resolution) { 111 DCHECK(CalledOnValidThread()); 112 113 // TODO(sergeyu): Move these checks to protocol layer. 114 if (!resolution.has_dips_width() || !resolution.has_dips_height() || 115 resolution.dips_width() < 0 || resolution.dips_height() < 0 || 116 resolution.width() <= 0 || resolution.height() <= 0) { 117 LOG(ERROR) << "Received invalid ClientResolution message."; 118 return; 119 } 120 121 VLOG(1) << "Received ClientResolution (dips_width=" 122 << resolution.dips_width() << ", dips_height=" 123 << resolution.dips_height() << ")"; 124 125 if (!screen_controls_) 126 return; 127 128 ScreenResolution client_resolution( 129 webrtc::DesktopSize(resolution.dips_width(), resolution.dips_height()), 130 webrtc::DesktopVector(kDefaultDPI, kDefaultDPI)); 131 132 // Try to match the client's resolution. 133 screen_controls_->SetScreenResolution(client_resolution); 134 } 135 136 void ClientSession::ControlVideo(const protocol::VideoControl& video_control) { 137 DCHECK(CalledOnValidThread()); 138 139 // Note that |video_scheduler_| may be NULL, depending upon whether extensions 140 // choose to wrap or "steal" the video capturer or encoder. 141 if (video_control.has_enable()) { 142 VLOG(1) << "Received VideoControl (enable=" 143 << video_control.enable() << ")"; 144 pause_video_ = !video_control.enable(); 145 if (video_scheduler_.get()) 146 video_scheduler_->Pause(pause_video_); 147 } 148 if (video_control.has_lossless_encode()) { 149 VLOG(1) << "Received VideoControl (lossless_encode=" 150 << video_control.lossless_encode() << ")"; 151 lossless_video_encode_ = video_control.lossless_encode(); 152 if (video_scheduler_.get()) 153 video_scheduler_->SetLosslessEncode(lossless_video_encode_); 154 } 155 if (video_control.has_lossless_color()) { 156 VLOG(1) << "Received VideoControl (lossless_color=" 157 << video_control.lossless_color() << ")"; 158 lossless_video_color_ = video_control.lossless_color(); 159 if (video_scheduler_.get()) 160 video_scheduler_->SetLosslessColor(lossless_video_color_); 161 } 162 } 163 164 void ClientSession::ControlAudio(const protocol::AudioControl& audio_control) { 165 DCHECK(CalledOnValidThread()); 166 167 if (audio_control.has_enable()) { 168 VLOG(1) << "Received AudioControl (enable=" 169 << audio_control.enable() << ")"; 170 if (audio_scheduler_.get()) 171 audio_scheduler_->Pause(!audio_control.enable()); 172 } 173 } 174 175 void ClientSession::SetCapabilities( 176 const protocol::Capabilities& capabilities) { 177 DCHECK(CalledOnValidThread()); 178 179 // Ignore all the messages but the 1st one. 180 if (client_capabilities_) { 181 LOG(WARNING) << "protocol::Capabilities has been received already."; 182 return; 183 } 184 185 // Compute the set of capabilities supported by both client and host. 186 client_capabilities_ = make_scoped_ptr(new std::string()); 187 if (capabilities.has_capabilities()) 188 *client_capabilities_ = capabilities.capabilities(); 189 capabilities_ = IntersectCapabilities(*client_capabilities_, 190 host_capabilities_); 191 extension_manager_->OnNegotiatedCapabilities( 192 connection_->client_stub(), capabilities_); 193 194 VLOG(1) << "Client capabilities: " << *client_capabilities_; 195 196 // Calculate the set of capabilities enabled by both client and host and 197 // pass it to the desktop environment if it is available. 198 desktop_environment_->SetCapabilities(capabilities_); 199 } 200 201 void ClientSession::RequestPairing( 202 const protocol::PairingRequest& pairing_request) { 203 if (pairing_registry_.get() && pairing_request.has_client_name()) { 204 protocol::PairingRegistry::Pairing pairing = 205 pairing_registry_->CreatePairing(pairing_request.client_name()); 206 protocol::PairingResponse pairing_response; 207 pairing_response.set_client_id(pairing.client_id()); 208 pairing_response.set_shared_secret(pairing.shared_secret()); 209 connection_->client_stub()->SetPairingResponse(pairing_response); 210 } 211 } 212 213 void ClientSession::DeliverClientMessage( 214 const protocol::ExtensionMessage& message) { 215 if (message.has_type()) { 216 if (message.type() == "test-echo") { 217 protocol::ExtensionMessage reply; 218 reply.set_type("test-echo-reply"); 219 if (message.has_data()) 220 reply.set_data(message.data().substr(0, 16)); 221 connection_->client_stub()->DeliverHostMessage(reply); 222 return; 223 } else if (message.type() == "gnubby-auth") { 224 if (gnubby_auth_handler_) { 225 gnubby_auth_handler_->DeliverClientMessage(message.data()); 226 } else { 227 HOST_LOG << "gnubby auth is not enabled"; 228 } 229 return; 230 } else { 231 if (!extension_manager_->OnExtensionMessage(message)) 232 return; 233 234 DLOG(INFO) << "Unexpected message received: " 235 << message.type() << ": " << message.data(); 236 } 237 } 238 } 239 240 void ClientSession::OnConnectionAuthenticating( 241 protocol::ConnectionToClient* connection) { 242 event_handler_->OnSessionAuthenticating(this); 243 } 244 245 void ClientSession::OnConnectionAuthenticated( 246 protocol::ConnectionToClient* connection) { 247 DCHECK(CalledOnValidThread()); 248 DCHECK_EQ(connection_.get(), connection); 249 DCHECK(!audio_scheduler_.get()); 250 DCHECK(!desktop_environment_); 251 DCHECK(!input_injector_); 252 DCHECK(!screen_controls_); 253 DCHECK(!video_scheduler_.get()); 254 255 auth_input_filter_.set_enabled(true); 256 auth_clipboard_filter_.set_enabled(true); 257 258 clipboard_echo_filter_.set_client_stub(connection_->client_stub()); 259 mouse_clamping_filter_.set_video_stub(connection_->video_stub()); 260 261 if (max_duration_ > base::TimeDelta()) { 262 // TODO(simonmorris): Let Disconnect() tell the client that the 263 // disconnection was caused by the session exceeding its maximum duration. 264 max_duration_timer_.Start(FROM_HERE, max_duration_, 265 this, &ClientSession::DisconnectSession); 266 } 267 268 // Disconnect the session if the connection was rejected by the host. 269 if (!event_handler_->OnSessionAuthenticated(this)) { 270 DisconnectSession(); 271 return; 272 } 273 274 // Create the desktop environment. Drop the connection if it could not be 275 // created for any reason (for instance the curtain could not initialize). 276 desktop_environment_ = 277 desktop_environment_factory_->Create(weak_factory_.GetWeakPtr()); 278 if (!desktop_environment_) { 279 DisconnectSession(); 280 return; 281 } 282 283 // Collate the set of capabilities to offer the client, if it supports them. 284 host_capabilities_ = desktop_environment_->GetCapabilities(); 285 if (!host_capabilities_.empty()) 286 host_capabilities_.append(" "); 287 host_capabilities_.append(extension_manager_->GetCapabilities()); 288 289 // Create the object that controls the screen resolution. 290 screen_controls_ = desktop_environment_->CreateScreenControls(); 291 292 // Create the event executor. 293 input_injector_ = desktop_environment_->CreateInputInjector(); 294 295 // Connect the host clipboard and input stubs. 296 host_input_filter_.set_input_stub(input_injector_.get()); 297 clipboard_echo_filter_.set_host_stub(input_injector_.get()); 298 299 // Create an AudioScheduler if audio is enabled, to pump audio samples. 300 if (connection_->session()->config().is_audio_enabled()) { 301 scoped_ptr<AudioEncoder> audio_encoder = 302 CreateAudioEncoder(connection_->session()->config()); 303 audio_scheduler_ = new AudioScheduler( 304 audio_task_runner_, 305 network_task_runner_, 306 desktop_environment_->CreateAudioCapturer(), 307 audio_encoder.Pass(), 308 connection_->audio_stub()); 309 } 310 311 // Create a GnubbyAuthHandler to proxy gnubbyd messages. 312 gnubby_auth_handler_ = desktop_environment_->CreateGnubbyAuthHandler( 313 connection_->client_stub()); 314 } 315 316 void ClientSession::OnConnectionChannelsConnected( 317 protocol::ConnectionToClient* connection) { 318 DCHECK(CalledOnValidThread()); 319 DCHECK_EQ(connection_.get(), connection); 320 321 // Negotiate capabilities with the client. 322 VLOG(1) << "Host capabilities: " << host_capabilities_; 323 324 protocol::Capabilities capabilities; 325 capabilities.set_capabilities(host_capabilities_); 326 connection_->client_stub()->SetCapabilities(capabilities); 327 328 // Start the event executor. 329 input_injector_->Start(CreateClipboardProxy()); 330 SetDisableInputs(false); 331 332 // Start recording video. 333 ResetVideoPipeline(); 334 335 // Start recording audio. 336 if (connection_->session()->config().is_audio_enabled()) 337 audio_scheduler_->Start(); 338 339 // Notify the event handler that all our channels are now connected. 340 event_handler_->OnSessionChannelsConnected(this); 341 } 342 343 void ClientSession::OnConnectionClosed( 344 protocol::ConnectionToClient* connection, 345 protocol::ErrorCode error) { 346 DCHECK(CalledOnValidThread()); 347 DCHECK_EQ(connection_.get(), connection); 348 349 // Ignore any further callbacks. 350 weak_factory_.InvalidateWeakPtrs(); 351 352 // If the client never authenticated then the session failed. 353 if (!auth_input_filter_.enabled()) 354 event_handler_->OnSessionAuthenticationFailed(this); 355 356 // Block any further input events from the client. 357 // TODO(wez): Fix ChromotingHost::OnSessionClosed not to check our 358 // is_authenticated(), so that we can disable |auth_*_filter_| here. 359 disable_input_filter_.set_enabled(false); 360 disable_clipboard_filter_.set_enabled(false); 361 362 // Ensure that any pressed keys or buttons are released. 363 input_tracker_.ReleaseAll(); 364 365 // Stop components access the client, audio or video stubs, which are no 366 // longer valid once ConnectionToClient calls OnConnectionClosed(). 367 if (audio_scheduler_.get()) { 368 audio_scheduler_->Stop(); 369 audio_scheduler_ = NULL; 370 } 371 if (video_scheduler_.get()) { 372 video_scheduler_->Stop(); 373 video_scheduler_ = NULL; 374 } 375 376 client_clipboard_factory_.InvalidateWeakPtrs(); 377 input_injector_.reset(); 378 screen_controls_.reset(); 379 desktop_environment_.reset(); 380 381 // Notify the ChromotingHost that this client is disconnected. 382 // TODO(sergeyu): Log failure reason? 383 event_handler_->OnSessionClosed(this); 384 } 385 386 void ClientSession::OnSequenceNumberUpdated( 387 protocol::ConnectionToClient* connection, int64 sequence_number) { 388 DCHECK(CalledOnValidThread()); 389 DCHECK_EQ(connection_.get(), connection); 390 391 if (video_scheduler_.get()) 392 video_scheduler_->UpdateSequenceNumber(sequence_number); 393 } 394 395 void ClientSession::OnRouteChange( 396 protocol::ConnectionToClient* connection, 397 const std::string& channel_name, 398 const protocol::TransportRoute& route) { 399 DCHECK(CalledOnValidThread()); 400 DCHECK_EQ(connection_.get(), connection); 401 event_handler_->OnSessionRouteChange(this, channel_name, route); 402 } 403 404 const std::string& ClientSession::client_jid() const { 405 return client_jid_; 406 } 407 408 void ClientSession::DisconnectSession() { 409 DCHECK(CalledOnValidThread()); 410 DCHECK(connection_.get()); 411 412 max_duration_timer_.Stop(); 413 414 // This triggers OnConnectionClosed(), and the session may be destroyed 415 // as the result, so this call must be the last in this method. 416 connection_->Disconnect(); 417 } 418 419 void ClientSession::OnLocalMouseMoved(const webrtc::DesktopVector& position) { 420 DCHECK(CalledOnValidThread()); 421 remote_input_filter_.LocalMouseMoved(position); 422 } 423 424 void ClientSession::SetDisableInputs(bool disable_inputs) { 425 DCHECK(CalledOnValidThread()); 426 427 if (disable_inputs) 428 input_tracker_.ReleaseAll(); 429 430 disable_input_filter_.set_enabled(!disable_inputs); 431 disable_clipboard_filter_.set_enabled(!disable_inputs); 432 } 433 434 void ClientSession::ResetVideoPipeline() { 435 DCHECK(CalledOnValidThread()); 436 437 if (video_scheduler_.get()) { 438 video_scheduler_->Stop(); 439 video_scheduler_ = NULL; 440 } 441 442 // Create VideoEncoder and DesktopCapturer to match the session's video 443 // channel configuration. 444 scoped_ptr<webrtc::DesktopCapturer> video_capturer = 445 desktop_environment_->CreateVideoCapturer(); 446 extension_manager_->OnCreateVideoCapturer(&video_capturer); 447 scoped_ptr<VideoEncoder> video_encoder = 448 CreateVideoEncoder(connection_->session()->config()); 449 extension_manager_->OnCreateVideoEncoder(&video_encoder); 450 451 // Don't start the VideoScheduler if either capturer or encoder are missing. 452 if (!video_capturer || !video_encoder) 453 return; 454 455 // Create a VideoScheduler to pump frames from the capturer to the client. 456 video_scheduler_ = new VideoScheduler( 457 video_capture_task_runner_, 458 video_encode_task_runner_, 459 network_task_runner_, 460 video_capturer.Pass(), 461 desktop_environment_->CreateMouseCursorMonitor(), 462 video_encoder.Pass(), 463 connection_->client_stub(), 464 &mouse_clamping_filter_); 465 466 // Apply video-control parameters to the new scheduler. 467 video_scheduler_->Pause(pause_video_); 468 video_scheduler_->SetLosslessEncode(lossless_video_encode_); 469 video_scheduler_->SetLosslessColor(lossless_video_color_); 470 471 // Start capturing the screen. 472 video_scheduler_->Start(); 473 } 474 475 void ClientSession::SetGnubbyAuthHandlerForTesting( 476 GnubbyAuthHandler* gnubby_auth_handler) { 477 DCHECK(CalledOnValidThread()); 478 gnubby_auth_handler_.reset(gnubby_auth_handler); 479 } 480 481 scoped_ptr<protocol::ClipboardStub> ClientSession::CreateClipboardProxy() { 482 DCHECK(CalledOnValidThread()); 483 484 return scoped_ptr<protocol::ClipboardStub>( 485 new protocol::ClipboardThreadProxy( 486 client_clipboard_factory_.GetWeakPtr(), 487 base::MessageLoopProxy::current())); 488 } 489 490 // TODO(sergeyu): Move this to SessionManager? 491 // static 492 scoped_ptr<VideoEncoder> ClientSession::CreateVideoEncoder( 493 const protocol::SessionConfig& config) { 494 const protocol::ChannelConfig& video_config = config.video_config(); 495 496 if (video_config.codec == protocol::ChannelConfig::CODEC_VP8) { 497 return remoting::VideoEncoderVpx::CreateForVP8().PassAs<VideoEncoder>(); 498 } else if (video_config.codec == protocol::ChannelConfig::CODEC_VP9) { 499 return remoting::VideoEncoderVpx::CreateForVP9().PassAs<VideoEncoder>(); 500 } else if (video_config.codec == protocol::ChannelConfig::CODEC_VERBATIM) { 501 return scoped_ptr<VideoEncoder>(new remoting::VideoEncoderVerbatim()); 502 } 503 504 NOTREACHED(); 505 return scoped_ptr<VideoEncoder>(); 506 } 507 508 // static 509 scoped_ptr<AudioEncoder> ClientSession::CreateAudioEncoder( 510 const protocol::SessionConfig& config) { 511 const protocol::ChannelConfig& audio_config = config.audio_config(); 512 513 if (audio_config.codec == protocol::ChannelConfig::CODEC_VERBATIM) { 514 return scoped_ptr<AudioEncoder>(new AudioEncoderVerbatim()); 515 } else if (audio_config.codec == protocol::ChannelConfig::CODEC_OPUS) { 516 return scoped_ptr<AudioEncoder>(new AudioEncoderOpus()); 517 } 518 519 NOTREACHED(); 520 return scoped_ptr<AudioEncoder>(); 521 } 522 523 } // namespace remoting 524
