1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "google_apis/gcm/engine/connection_factory_impl.h" 6 7 #include <cmath> 8 9 #include "base/message_loop/message_loop.h" 10 #include "base/run_loop.h" 11 #include "base/test/simple_test_tick_clock.h" 12 #include "google_apis/gcm/base/mcs_util.h" 13 #include "google_apis/gcm/engine/fake_connection_handler.h" 14 #include "google_apis/gcm/monitoring/fake_gcm_stats_recorder.h" 15 #include "net/base/backoff_entry.h" 16 #include "net/http/http_network_session.h" 17 #include "testing/gtest/include/gtest/gtest.h" 18 19 class Policy; 20 21 namespace gcm { 22 namespace { 23 24 const char kMCSEndpoint[] = "http://my.server"; 25 const char kMCSEndpoint2[] = "http://my.alt.server"; 26 27 const int kBackoffDelayMs = 1; 28 const int kBackoffMultiplier = 2; 29 30 // A backoff policy with small enough delays that tests aren't burdened. 31 const net::BackoffEntry::Policy kTestBackoffPolicy = { 32 // Number of initial errors (in sequence) to ignore before applying 33 // exponential back-off rules. 34 0, 35 36 // Initial delay for exponential back-off in ms. 37 kBackoffDelayMs, 38 39 // Factor by which the waiting time will be multiplied. 40 kBackoffMultiplier, 41 42 // Fuzzing percentage. ex: 10% will spread requests randomly 43 // between 90%-100% of the calculated time. 44 0, 45 46 // Maximum amount of time we are willing to delay our request in ms. 47 10, 48 49 // Time to keep an entry from being discarded even when it 50 // has no significant state, -1 to never discard. 51 -1, 52 53 // Don't use initial delay unless the last request was an error. 54 false, 55 }; 56 57 std::vector<GURL> BuildEndpoints() { 58 std::vector<GURL> endpoints; 59 endpoints.push_back(GURL(kMCSEndpoint)); 60 endpoints.push_back(GURL(kMCSEndpoint2)); 61 return endpoints; 62 } 63 64 // Helper for calculating total expected exponential backoff delay given an 65 // arbitrary number of failed attempts. See BackoffEntry::CalculateReleaseTime. 66 double CalculateBackoff(int num_attempts) { 67 double delay = kBackoffDelayMs; 68 for (int i = 1; i < num_attempts; ++i) { 69 delay += kBackoffDelayMs * pow(static_cast<double>(kBackoffMultiplier), 70 i - 1); 71 } 72 DVLOG(1) << "Expected backoff " << delay << " milliseconds."; 73 return delay; 74 } 75 76 void ReadContinuation( 77 scoped_ptr<google::protobuf::MessageLite> message) { 78 } 79 80 void WriteContinuation() { 81 } 82 83 class TestBackoffEntry : public net::BackoffEntry { 84 public: 85 explicit TestBackoffEntry(base::SimpleTestTickClock* tick_clock); 86 virtual ~TestBackoffEntry(); 87 88 virtual base::TimeTicks ImplGetTimeNow() const OVERRIDE; 89 90 private: 91 base::SimpleTestTickClock* tick_clock_; 92 }; 93 94 TestBackoffEntry::TestBackoffEntry(base::SimpleTestTickClock* tick_clock) 95 : BackoffEntry(&kTestBackoffPolicy), 96 tick_clock_(tick_clock) { 97 } 98 99 TestBackoffEntry::~TestBackoffEntry() {} 100 101 base::TimeTicks TestBackoffEntry::ImplGetTimeNow() const { 102 return tick_clock_->NowTicks(); 103 } 104 105 // A connection factory that stubs out network requests and overrides the 106 // backoff policy. 107 class TestConnectionFactoryImpl : public ConnectionFactoryImpl { 108 public: 109 TestConnectionFactoryImpl(const base::Closure& finished_callback); 110 virtual ~TestConnectionFactoryImpl(); 111 112 void InitializeFactory(); 113 114 // Overridden stubs. 115 virtual void ConnectImpl() OVERRIDE; 116 virtual void InitHandler() OVERRIDE; 117 virtual scoped_ptr<net::BackoffEntry> CreateBackoffEntry( 118 const net::BackoffEntry::Policy* const policy) OVERRIDE; 119 virtual scoped_ptr<ConnectionHandler> CreateConnectionHandler( 120 base::TimeDelta read_timeout, 121 const ConnectionHandler::ProtoReceivedCallback& read_callback, 122 const ConnectionHandler::ProtoSentCallback& write_callback, 123 const ConnectionHandler::ConnectionChangedCallback& connection_callback) 124 OVERRIDE; 125 virtual base::TimeTicks NowTicks() OVERRIDE; 126 127 // Helpers for verifying connection attempts are made. Connection results 128 // must be consumed. 129 void SetConnectResult(int connect_result); 130 void SetMultipleConnectResults(int connect_result, int num_expected_attempts); 131 132 // Force a login handshake to be delayed. 133 void SetDelayLogin(bool delay_login); 134 135 base::SimpleTestTickClock* tick_clock() { return &tick_clock_; } 136 137 private: 138 // Clock for controlling delay. 139 base::SimpleTestTickClock tick_clock_; 140 // The result to return on the next connect attempt. 141 int connect_result_; 142 // The number of expected connection attempts; 143 int num_expected_attempts_; 144 // Whether all expected connection attempts have been fulfilled since an 145 // expectation was last set. 146 bool connections_fulfilled_; 147 // Whether to delay a login handshake completion or not. 148 bool delay_login_; 149 // Callback to invoke when all connection attempts have been made. 150 base::Closure finished_callback_; 151 // The current fake connection handler.. 152 FakeConnectionHandler* fake_handler_; 153 FakeGCMStatsRecorder dummy_recorder_; 154 }; 155 156 TestConnectionFactoryImpl::TestConnectionFactoryImpl( 157 const base::Closure& finished_callback) 158 : ConnectionFactoryImpl(BuildEndpoints(), 159 net::BackoffEntry::Policy(), 160 NULL, 161 NULL, 162 &dummy_recorder_), 163 connect_result_(net::ERR_UNEXPECTED), 164 num_expected_attempts_(0), 165 connections_fulfilled_(true), 166 delay_login_(false), 167 finished_callback_(finished_callback), 168 fake_handler_(NULL) { 169 // Set a non-null time. 170 tick_clock_.Advance(base::TimeDelta::FromMilliseconds(1)); 171 } 172 173 TestConnectionFactoryImpl::~TestConnectionFactoryImpl() { 174 EXPECT_EQ(0, num_expected_attempts_); 175 } 176 177 void TestConnectionFactoryImpl::ConnectImpl() { 178 ASSERT_GT(num_expected_attempts_, 0); 179 scoped_ptr<mcs_proto::LoginRequest> request(BuildLoginRequest(0, 0, "")); 180 GetConnectionHandler()->Init(*request, NULL); 181 OnConnectDone(connect_result_); 182 if (!NextRetryAttempt().is_null()) { 183 // Advance the time to the next retry time. 184 base::TimeDelta time_till_retry = 185 NextRetryAttempt() - tick_clock_.NowTicks(); 186 tick_clock_.Advance(time_till_retry); 187 } 188 --num_expected_attempts_; 189 if (num_expected_attempts_ == 0) { 190 connect_result_ = net::ERR_UNEXPECTED; 191 connections_fulfilled_ = true; 192 finished_callback_.Run(); 193 } 194 } 195 196 void TestConnectionFactoryImpl::InitHandler() { 197 EXPECT_NE(connect_result_, net::ERR_UNEXPECTED); 198 if (!delay_login_) 199 ConnectionHandlerCallback(net::OK); 200 } 201 202 scoped_ptr<net::BackoffEntry> TestConnectionFactoryImpl::CreateBackoffEntry( 203 const net::BackoffEntry::Policy* const policy) { 204 return scoped_ptr<net::BackoffEntry>(new TestBackoffEntry(&tick_clock_)); 205 } 206 207 scoped_ptr<ConnectionHandler> 208 TestConnectionFactoryImpl::CreateConnectionHandler( 209 base::TimeDelta read_timeout, 210 const ConnectionHandler::ProtoReceivedCallback& read_callback, 211 const ConnectionHandler::ProtoSentCallback& write_callback, 212 const ConnectionHandler::ConnectionChangedCallback& connection_callback) { 213 fake_handler_ = new FakeConnectionHandler( 214 base::Bind(&ReadContinuation), 215 base::Bind(&WriteContinuation)); 216 return make_scoped_ptr<ConnectionHandler>(fake_handler_); 217 } 218 219 base::TimeTicks TestConnectionFactoryImpl::NowTicks() { 220 return tick_clock_.NowTicks(); 221 } 222 223 void TestConnectionFactoryImpl::SetConnectResult(int connect_result) { 224 DCHECK_NE(connect_result, net::ERR_UNEXPECTED); 225 ASSERT_EQ(0, num_expected_attempts_); 226 connections_fulfilled_ = false; 227 connect_result_ = connect_result; 228 num_expected_attempts_ = 1; 229 fake_handler_->ExpectOutgoingMessage( 230 MCSMessage(kLoginRequestTag, 231 BuildLoginRequest(0, 0, "").PassAs< 232 const google::protobuf::MessageLite>())); 233 } 234 235 void TestConnectionFactoryImpl::SetMultipleConnectResults( 236 int connect_result, 237 int num_expected_attempts) { 238 DCHECK_NE(connect_result, net::ERR_UNEXPECTED); 239 DCHECK_GT(num_expected_attempts, 0); 240 ASSERT_EQ(0, num_expected_attempts_); 241 connections_fulfilled_ = false; 242 connect_result_ = connect_result; 243 num_expected_attempts_ = num_expected_attempts; 244 for (int i = 0 ; i < num_expected_attempts; ++i) { 245 fake_handler_->ExpectOutgoingMessage( 246 MCSMessage(kLoginRequestTag, 247 BuildLoginRequest(0, 0, "").PassAs< 248 const google::protobuf::MessageLite>())); 249 } 250 } 251 252 void TestConnectionFactoryImpl::SetDelayLogin(bool delay_login) { 253 delay_login_ = delay_login; 254 fake_handler_->set_fail_login(delay_login_); 255 } 256 257 } // namespace 258 259 class ConnectionFactoryImplTest 260 : public testing::Test, 261 public ConnectionFactory::ConnectionListener { 262 public: 263 ConnectionFactoryImplTest(); 264 virtual ~ConnectionFactoryImplTest(); 265 266 TestConnectionFactoryImpl* factory() { return &factory_; } 267 GURL& connected_server() { return connected_server_; } 268 269 void WaitForConnections(); 270 271 // ConnectionFactory::ConnectionListener 272 virtual void OnConnected(const GURL& current_server, 273 const net::IPEndPoint& ip_endpoint) OVERRIDE; 274 virtual void OnDisconnected() OVERRIDE; 275 276 private: 277 void ConnectionsComplete(); 278 279 TestConnectionFactoryImpl factory_; 280 base::MessageLoop message_loop_; 281 scoped_ptr<base::RunLoop> run_loop_; 282 283 GURL connected_server_; 284 }; 285 286 ConnectionFactoryImplTest::ConnectionFactoryImplTest() 287 : factory_(base::Bind(&ConnectionFactoryImplTest::ConnectionsComplete, 288 base::Unretained(this))), 289 run_loop_(new base::RunLoop()) { 290 factory()->SetConnectionListener(this); 291 factory()->Initialize( 292 ConnectionFactory::BuildLoginRequestCallback(), 293 ConnectionHandler::ProtoReceivedCallback(), 294 ConnectionHandler::ProtoSentCallback()); 295 } 296 ConnectionFactoryImplTest::~ConnectionFactoryImplTest() {} 297 298 void ConnectionFactoryImplTest::WaitForConnections() { 299 run_loop_->Run(); 300 run_loop_.reset(new base::RunLoop()); 301 } 302 303 void ConnectionFactoryImplTest::ConnectionsComplete() { 304 if (!run_loop_) 305 return; 306 run_loop_->Quit(); 307 } 308 309 void ConnectionFactoryImplTest::OnConnected( 310 const GURL& current_server, 311 const net::IPEndPoint& ip_endpoint) { 312 connected_server_ = current_server; 313 } 314 315 void ConnectionFactoryImplTest::OnDisconnected() { 316 connected_server_ = GURL(); 317 } 318 319 // Verify building a connection handler works. 320 TEST_F(ConnectionFactoryImplTest, Initialize) { 321 ConnectionHandler* handler = factory()->GetConnectionHandler(); 322 ASSERT_TRUE(handler); 323 EXPECT_FALSE(factory()->IsEndpointReachable()); 324 EXPECT_FALSE(connected_server().is_valid()); 325 } 326 327 // An initial successful connection should not result in backoff. 328 TEST_F(ConnectionFactoryImplTest, ConnectSuccess) { 329 factory()->SetConnectResult(net::OK); 330 factory()->Connect(); 331 EXPECT_TRUE(factory()->NextRetryAttempt().is_null()); 332 EXPECT_EQ(factory()->GetCurrentEndpoint(), BuildEndpoints()[0]); 333 EXPECT_TRUE(factory()->IsEndpointReachable()); 334 EXPECT_TRUE(connected_server().is_valid()); 335 } 336 337 // A connection failure should result in backoff, and attempting the fallback 338 // endpoint next. 339 TEST_F(ConnectionFactoryImplTest, ConnectFail) { 340 factory()->SetConnectResult(net::ERR_CONNECTION_FAILED); 341 factory()->Connect(); 342 EXPECT_FALSE(factory()->NextRetryAttempt().is_null()); 343 EXPECT_EQ(factory()->GetCurrentEndpoint(), BuildEndpoints()[1]); 344 EXPECT_FALSE(factory()->IsEndpointReachable()); 345 EXPECT_FALSE(connected_server().is_valid()); 346 } 347 348 // A connection success after a failure should reset backoff. 349 TEST_F(ConnectionFactoryImplTest, FailThenSucceed) { 350 factory()->SetConnectResult(net::ERR_CONNECTION_FAILED); 351 base::TimeTicks connect_time = factory()->tick_clock()->NowTicks(); 352 factory()->Connect(); 353 WaitForConnections(); 354 EXPECT_FALSE(factory()->IsEndpointReachable()); 355 EXPECT_FALSE(connected_server().is_valid()); 356 base::TimeTicks retry_time = factory()->NextRetryAttempt(); 357 EXPECT_FALSE(retry_time.is_null()); 358 EXPECT_GE((retry_time - connect_time).InMilliseconds(), CalculateBackoff(1)); 359 factory()->SetConnectResult(net::OK); 360 WaitForConnections(); 361 EXPECT_TRUE(factory()->NextRetryAttempt().is_null()); 362 EXPECT_TRUE(factory()->IsEndpointReachable()); 363 EXPECT_TRUE(connected_server().is_valid()); 364 } 365 366 // Multiple connection failures should retry with an exponentially increasing 367 // backoff, then reset on success. 368 TEST_F(ConnectionFactoryImplTest, MultipleFailuresThenSucceed) { 369 const int kNumAttempts = 5; 370 factory()->SetMultipleConnectResults(net::ERR_CONNECTION_FAILED, 371 kNumAttempts); 372 373 base::TimeTicks connect_time = factory()->tick_clock()->NowTicks(); 374 factory()->Connect(); 375 WaitForConnections(); 376 EXPECT_FALSE(factory()->IsEndpointReachable()); 377 EXPECT_FALSE(connected_server().is_valid()); 378 base::TimeTicks retry_time = factory()->NextRetryAttempt(); 379 EXPECT_FALSE(retry_time.is_null()); 380 EXPECT_GE((retry_time - connect_time).InMilliseconds(), 381 CalculateBackoff(kNumAttempts)); 382 383 factory()->SetConnectResult(net::OK); 384 WaitForConnections(); 385 EXPECT_TRUE(factory()->NextRetryAttempt().is_null()); 386 EXPECT_TRUE(factory()->IsEndpointReachable()); 387 EXPECT_TRUE(connected_server().is_valid()); 388 } 389 390 // Network change events should trigger canary connections. 391 TEST_F(ConnectionFactoryImplTest, FailThenNetworkChangeEvent) { 392 factory()->SetConnectResult(net::ERR_CONNECTION_FAILED); 393 factory()->Connect(); 394 WaitForConnections(); 395 base::TimeTicks initial_backoff = factory()->NextRetryAttempt(); 396 EXPECT_FALSE(initial_backoff.is_null()); 397 398 factory()->SetConnectResult(net::ERR_FAILED); 399 factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_WIFI); 400 WaitForConnections(); 401 402 // Backoff should increase. 403 base::TimeTicks next_backoff = factory()->NextRetryAttempt(); 404 EXPECT_GT(next_backoff, initial_backoff); 405 EXPECT_FALSE(factory()->IsEndpointReachable()); 406 } 407 408 // Verify that we reconnect even if a canary succeeded then disconnected while 409 // a backoff was pending. 410 TEST_F(ConnectionFactoryImplTest, CanarySucceedsThenDisconnects) { 411 factory()->SetConnectResult(net::ERR_CONNECTION_FAILED); 412 factory()->Connect(); 413 WaitForConnections(); 414 base::TimeTicks initial_backoff = factory()->NextRetryAttempt(); 415 EXPECT_FALSE(initial_backoff.is_null()); 416 417 factory()->SetConnectResult(net::OK); 418 factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_ETHERNET); 419 WaitForConnections(); 420 EXPECT_TRUE(factory()->IsEndpointReachable()); 421 EXPECT_TRUE(connected_server().is_valid()); 422 423 factory()->SetConnectResult(net::OK); 424 factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE); 425 EXPECT_FALSE(factory()->IsEndpointReachable()); 426 EXPECT_FALSE(connected_server().is_valid()); 427 WaitForConnections(); 428 EXPECT_TRUE(factory()->IsEndpointReachable()); 429 EXPECT_TRUE(connected_server().is_valid()); 430 } 431 432 // Verify that if a canary connects, but hasn't finished the handshake, a 433 // pending backoff attempt doesn't interrupt the connection. 434 TEST_F(ConnectionFactoryImplTest, CanarySucceedsRetryDuringLogin) { 435 factory()->SetConnectResult(net::ERR_CONNECTION_FAILED); 436 factory()->Connect(); 437 WaitForConnections(); 438 base::TimeTicks initial_backoff = factory()->NextRetryAttempt(); 439 EXPECT_FALSE(initial_backoff.is_null()); 440 441 factory()->SetDelayLogin(true); 442 factory()->SetConnectResult(net::OK); 443 factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_WIFI); 444 WaitForConnections(); 445 EXPECT_FALSE(factory()->IsEndpointReachable()); 446 447 // Pump the loop, to ensure the pending backoff retry has no effect. 448 base::MessageLoop::current()->PostDelayedTask( 449 FROM_HERE, 450 base::MessageLoop::QuitClosure(), 451 base::TimeDelta::FromMilliseconds(1)); 452 WaitForConnections(); 453 } 454 455 // Fail after successful connection via signal reset. 456 TEST_F(ConnectionFactoryImplTest, FailViaSignalReset) { 457 factory()->SetConnectResult(net::OK); 458 factory()->Connect(); 459 EXPECT_TRUE(factory()->NextRetryAttempt().is_null()); 460 461 factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE); 462 EXPECT_FALSE(factory()->NextRetryAttempt().is_null()); 463 EXPECT_FALSE(factory()->IsEndpointReachable()); 464 } 465 466 TEST_F(ConnectionFactoryImplTest, IgnoreResetWhileConnecting) { 467 factory()->SetConnectResult(net::OK); 468 factory()->Connect(); 469 EXPECT_TRUE(factory()->NextRetryAttempt().is_null()); 470 471 factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE); 472 base::TimeTicks retry_time = factory()->NextRetryAttempt(); 473 EXPECT_FALSE(retry_time.is_null()); 474 EXPECT_FALSE(factory()->IsEndpointReachable()); 475 476 const int kNumAttempts = 5; 477 for (int i = 0; i < kNumAttempts; ++i) 478 factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE); 479 EXPECT_EQ(retry_time, factory()->NextRetryAttempt()); 480 EXPECT_FALSE(factory()->IsEndpointReachable()); 481 } 482 483 // Go into backoff due to connection failure. On successful connection, receive 484 // a signal reset. The original backoff should be restored and extended, rather 485 // than a new backoff starting from scratch. 486 TEST_F(ConnectionFactoryImplTest, SignalResetRestoresBackoff) { 487 factory()->SetConnectResult(net::ERR_CONNECTION_FAILED); 488 base::TimeTicks connect_time = factory()->tick_clock()->NowTicks(); 489 factory()->Connect(); 490 WaitForConnections(); 491 base::TimeTicks retry_time = factory()->NextRetryAttempt(); 492 EXPECT_FALSE(retry_time.is_null()); 493 494 factory()->SetConnectResult(net::OK); 495 connect_time = factory()->tick_clock()->NowTicks(); 496 WaitForConnections(); 497 EXPECT_TRUE(factory()->NextRetryAttempt().is_null()); 498 499 factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE); 500 EXPECT_FALSE(factory()->IsEndpointReachable()); 501 EXPECT_FALSE(connected_server().is_valid()); 502 EXPECT_NE(retry_time, factory()->NextRetryAttempt()); 503 retry_time = factory()->NextRetryAttempt(); 504 EXPECT_FALSE(retry_time.is_null()); 505 EXPECT_GE((retry_time - connect_time).InMilliseconds(), 506 CalculateBackoff(2)); 507 508 factory()->SetConnectResult(net::OK); 509 connect_time = factory()->tick_clock()->NowTicks(); 510 factory()->tick_clock()->Advance( 511 factory()->NextRetryAttempt() - connect_time); 512 WaitForConnections(); 513 EXPECT_TRUE(factory()->NextRetryAttempt().is_null()); 514 EXPECT_TRUE(factory()->IsEndpointReachable()); 515 EXPECT_TRUE(connected_server().is_valid()); 516 517 factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE); 518 EXPECT_NE(retry_time, factory()->NextRetryAttempt()); 519 retry_time = factory()->NextRetryAttempt(); 520 EXPECT_FALSE(retry_time.is_null()); 521 EXPECT_GE((retry_time - connect_time).InMilliseconds(), 522 CalculateBackoff(3)); 523 EXPECT_FALSE(factory()->IsEndpointReachable()); 524 EXPECT_FALSE(connected_server().is_valid()); 525 } 526 527 // When the network is disconnected, close the socket and suppress further 528 // connection attempts until the network returns. 529 // Disabled while crbug.com/396687 is being investigated. 530 TEST_F(ConnectionFactoryImplTest, DISABLED_SuppressConnectWhenNoNetwork) { 531 factory()->SetConnectResult(net::OK); 532 factory()->Connect(); 533 EXPECT_TRUE(factory()->NextRetryAttempt().is_null()); 534 EXPECT_TRUE(factory()->IsEndpointReachable()); 535 536 // Advance clock so the login window reset isn't encountered. 537 factory()->tick_clock()->Advance(base::TimeDelta::FromSeconds(11)); 538 539 // Will trigger reset, but will not attempt a new connection. 540 factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_NONE); 541 EXPECT_FALSE(factory()->IsEndpointReachable()); 542 EXPECT_TRUE(factory()->NextRetryAttempt().is_null()); 543 544 // When the network returns, attempt to connect. 545 factory()->SetConnectResult(net::OK); 546 factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_4G); 547 WaitForConnections(); 548 549 EXPECT_TRUE(factory()->IsEndpointReachable()); 550 EXPECT_TRUE(factory()->NextRetryAttempt().is_null()); 551 } 552 553 } // namespace gcm 554