Home | History | Annotate | Download | only in engine 
      1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "google_apis/gcm/engine/connection_factory_impl.h"
      6 
      7 #include <cmath>
      8 
      9 #include "base/message_loop/message_loop.h"
     10 #include "base/run_loop.h"
     11 #include "base/test/simple_test_tick_clock.h"
     12 #include "google_apis/gcm/base/mcs_util.h"
     13 #include "google_apis/gcm/engine/fake_connection_handler.h"
     14 #include "google_apis/gcm/monitoring/fake_gcm_stats_recorder.h"
     15 #include "net/base/backoff_entry.h"
     16 #include "net/http/http_network_session.h"
     17 #include "testing/gtest/include/gtest/gtest.h"
     18 
     19 class Policy;
     20 
     21 namespace gcm {
     22 namespace {
     23 
     24 const char kMCSEndpoint[] = "http://my.server";
     25 const char kMCSEndpoint2[] = "http://my.alt.server";
     26 
     27 const int kBackoffDelayMs = 1;
     28 const int kBackoffMultiplier = 2;
     29 
     30 // A backoff policy with small enough delays that tests aren't burdened.
     31 const net::BackoffEntry::Policy kTestBackoffPolicy = {
     32   // Number of initial errors (in sequence) to ignore before applying
     33   // exponential back-off rules.
     34   0,
     35 
     36   // Initial delay for exponential back-off in ms.
     37   kBackoffDelayMs,
     38 
     39   // Factor by which the waiting time will be multiplied.
     40   kBackoffMultiplier,
     41 
     42   // Fuzzing percentage. ex: 10% will spread requests randomly
     43   // between 90%-100% of the calculated time.
     44   0,
     45 
     46   // Maximum amount of time we are willing to delay our request in ms.
     47   10,
     48 
     49   // Time to keep an entry from being discarded even when it
     50   // has no significant state, -1 to never discard.
     51   -1,
     52 
     53   // Don't use initial delay unless the last request was an error.
     54   false,
     55 };
     56 
     57 std::vector<GURL> BuildEndpoints() {
     58   std::vector<GURL> endpoints;
     59   endpoints.push_back(GURL(kMCSEndpoint));
     60   endpoints.push_back(GURL(kMCSEndpoint2));
     61   return endpoints;
     62 }
     63 
     64 // Helper for calculating total expected exponential backoff delay given an
     65 // arbitrary number of failed attempts. See BackoffEntry::CalculateReleaseTime.
     66 double CalculateBackoff(int num_attempts) {
     67   double delay = kBackoffDelayMs;
     68   for (int i = 1; i < num_attempts; ++i) {
     69     delay += kBackoffDelayMs * pow(static_cast<double>(kBackoffMultiplier),
     70                                    i - 1);
     71   }
     72   DVLOG(1) << "Expected backoff " << delay << " milliseconds.";
     73   return delay;
     74 }
     75 
     76 void ReadContinuation(
     77     scoped_ptr<google::protobuf::MessageLite> message) {
     78 }
     79 
     80 void WriteContinuation() {
     81 }
     82 
     83 class TestBackoffEntry : public net::BackoffEntry {
     84  public:
     85   explicit TestBackoffEntry(base::SimpleTestTickClock* tick_clock);
     86   virtual ~TestBackoffEntry();
     87 
     88   virtual base::TimeTicks ImplGetTimeNow() const OVERRIDE;
     89 
     90  private:
     91   base::SimpleTestTickClock* tick_clock_;
     92 };
     93 
     94 TestBackoffEntry::TestBackoffEntry(base::SimpleTestTickClock* tick_clock)
     95     : BackoffEntry(&kTestBackoffPolicy),
     96       tick_clock_(tick_clock) {
     97 }
     98 
     99 TestBackoffEntry::~TestBackoffEntry() {}
    100 
    101 base::TimeTicks TestBackoffEntry::ImplGetTimeNow() const {
    102   return tick_clock_->NowTicks();
    103 }
    104 
    105 // A connection factory that stubs out network requests and overrides the
    106 // backoff policy.
    107 class TestConnectionFactoryImpl : public ConnectionFactoryImpl {
    108  public:
    109   TestConnectionFactoryImpl(const base::Closure& finished_callback);
    110   virtual ~TestConnectionFactoryImpl();
    111 
    112   void InitializeFactory();
    113 
    114   // Overridden stubs.
    115   virtual void ConnectImpl() OVERRIDE;
    116   virtual void InitHandler() OVERRIDE;
    117   virtual scoped_ptr<net::BackoffEntry> CreateBackoffEntry(
    118       const net::BackoffEntry::Policy* const policy) OVERRIDE;
    119   virtual scoped_ptr<ConnectionHandler> CreateConnectionHandler(
    120       base::TimeDelta read_timeout,
    121       const ConnectionHandler::ProtoReceivedCallback& read_callback,
    122       const ConnectionHandler::ProtoSentCallback& write_callback,
    123       const ConnectionHandler::ConnectionChangedCallback& connection_callback)
    124           OVERRIDE;
    125   virtual base::TimeTicks NowTicks() OVERRIDE;
    126 
    127   // Helpers for verifying connection attempts are made. Connection results
    128   // must be consumed.
    129   void SetConnectResult(int connect_result);
    130   void SetMultipleConnectResults(int connect_result, int num_expected_attempts);
    131 
    132   // Force a login handshake to be delayed.
    133   void SetDelayLogin(bool delay_login);
    134 
    135   base::SimpleTestTickClock* tick_clock() { return &tick_clock_; }
    136 
    137  private:
    138   // Clock for controlling delay.
    139   base::SimpleTestTickClock tick_clock_;
    140   // The result to return on the next connect attempt.
    141   int connect_result_;
    142   // The number of expected connection attempts;
    143   int num_expected_attempts_;
    144   // Whether all expected connection attempts have been fulfilled since an
    145   // expectation was last set.
    146   bool connections_fulfilled_;
    147   // Whether to delay a login handshake completion or not.
    148   bool delay_login_;
    149   // Callback to invoke when all connection attempts have been made.
    150   base::Closure finished_callback_;
    151   // The current fake connection handler..
    152   FakeConnectionHandler* fake_handler_;
    153   FakeGCMStatsRecorder dummy_recorder_;
    154 };
    155 
    156 TestConnectionFactoryImpl::TestConnectionFactoryImpl(
    157     const base::Closure& finished_callback)
    158     : ConnectionFactoryImpl(BuildEndpoints(),
    159                             net::BackoffEntry::Policy(),
    160                             NULL,
    161                             NULL,
    162                             &dummy_recorder_),
    163       connect_result_(net::ERR_UNEXPECTED),
    164       num_expected_attempts_(0),
    165       connections_fulfilled_(true),
    166       delay_login_(false),
    167       finished_callback_(finished_callback),
    168       fake_handler_(NULL) {
    169   // Set a non-null time.
    170   tick_clock_.Advance(base::TimeDelta::FromMilliseconds(1));
    171 }
    172 
    173 TestConnectionFactoryImpl::~TestConnectionFactoryImpl() {
    174   EXPECT_EQ(0, num_expected_attempts_);
    175 }
    176 
    177 void TestConnectionFactoryImpl::ConnectImpl() {
    178   ASSERT_GT(num_expected_attempts_, 0);
    179   scoped_ptr<mcs_proto::LoginRequest> request(BuildLoginRequest(0, 0, ""));
    180   GetConnectionHandler()->Init(*request, NULL);
    181   OnConnectDone(connect_result_);
    182   if (!NextRetryAttempt().is_null()) {
    183     // Advance the time to the next retry time.
    184     base::TimeDelta time_till_retry =
    185         NextRetryAttempt() - tick_clock_.NowTicks();
    186     tick_clock_.Advance(time_till_retry);
    187   }
    188   --num_expected_attempts_;
    189   if (num_expected_attempts_ == 0) {
    190     connect_result_ = net::ERR_UNEXPECTED;
    191     connections_fulfilled_ = true;
    192     finished_callback_.Run();
    193   }
    194 }
    195 
    196 void TestConnectionFactoryImpl::InitHandler() {
    197   EXPECT_NE(connect_result_, net::ERR_UNEXPECTED);
    198   if (!delay_login_)
    199     ConnectionHandlerCallback(net::OK);
    200 }
    201 
    202 scoped_ptr<net::BackoffEntry> TestConnectionFactoryImpl::CreateBackoffEntry(
    203     const net::BackoffEntry::Policy* const policy) {
    204   return scoped_ptr<net::BackoffEntry>(new TestBackoffEntry(&tick_clock_));
    205 }
    206 
    207 scoped_ptr<ConnectionHandler>
    208 TestConnectionFactoryImpl::CreateConnectionHandler(
    209     base::TimeDelta read_timeout,
    210     const ConnectionHandler::ProtoReceivedCallback& read_callback,
    211     const ConnectionHandler::ProtoSentCallback& write_callback,
    212     const ConnectionHandler::ConnectionChangedCallback& connection_callback) {
    213   fake_handler_ = new FakeConnectionHandler(
    214       base::Bind(&ReadContinuation),
    215       base::Bind(&WriteContinuation));
    216   return make_scoped_ptr<ConnectionHandler>(fake_handler_);
    217 }
    218 
    219 base::TimeTicks TestConnectionFactoryImpl::NowTicks() {
    220   return tick_clock_.NowTicks();
    221 }
    222 
    223 void TestConnectionFactoryImpl::SetConnectResult(int connect_result) {
    224   DCHECK_NE(connect_result, net::ERR_UNEXPECTED);
    225   ASSERT_EQ(0, num_expected_attempts_);
    226   connections_fulfilled_ = false;
    227   connect_result_ = connect_result;
    228   num_expected_attempts_ = 1;
    229   fake_handler_->ExpectOutgoingMessage(
    230       MCSMessage(kLoginRequestTag,
    231                  BuildLoginRequest(0, 0, "").PassAs<
    232                      const google::protobuf::MessageLite>()));
    233 }
    234 
    235 void TestConnectionFactoryImpl::SetMultipleConnectResults(
    236     int connect_result,
    237     int num_expected_attempts) {
    238   DCHECK_NE(connect_result, net::ERR_UNEXPECTED);
    239   DCHECK_GT(num_expected_attempts, 0);
    240   ASSERT_EQ(0, num_expected_attempts_);
    241   connections_fulfilled_ = false;
    242   connect_result_ = connect_result;
    243   num_expected_attempts_ = num_expected_attempts;
    244   for (int i = 0 ; i < num_expected_attempts; ++i) {
    245     fake_handler_->ExpectOutgoingMessage(
    246         MCSMessage(kLoginRequestTag,
    247                    BuildLoginRequest(0, 0, "").PassAs<
    248                        const google::protobuf::MessageLite>()));
    249   }
    250 }
    251 
    252 void TestConnectionFactoryImpl::SetDelayLogin(bool delay_login) {
    253   delay_login_ = delay_login;
    254   fake_handler_->set_fail_login(delay_login_);
    255 }
    256 
    257 }  // namespace
    258 
    259 class ConnectionFactoryImplTest
    260     : public testing::Test,
    261       public ConnectionFactory::ConnectionListener {
    262  public:
    263   ConnectionFactoryImplTest();
    264   virtual ~ConnectionFactoryImplTest();
    265 
    266   TestConnectionFactoryImpl* factory() { return &factory_; }
    267   GURL& connected_server() { return connected_server_; }
    268 
    269   void WaitForConnections();
    270 
    271   // ConnectionFactory::ConnectionListener
    272   virtual void OnConnected(const GURL& current_server,
    273                            const net::IPEndPoint& ip_endpoint) OVERRIDE;
    274   virtual void OnDisconnected() OVERRIDE;
    275 
    276  private:
    277   void ConnectionsComplete();
    278 
    279   TestConnectionFactoryImpl factory_;
    280   base::MessageLoop message_loop_;
    281   scoped_ptr<base::RunLoop> run_loop_;
    282 
    283   GURL connected_server_;
    284 };
    285 
    286 ConnectionFactoryImplTest::ConnectionFactoryImplTest()
    287    : factory_(base::Bind(&ConnectionFactoryImplTest::ConnectionsComplete,
    288                          base::Unretained(this))),
    289      run_loop_(new base::RunLoop()) {
    290   factory()->SetConnectionListener(this);
    291   factory()->Initialize(
    292       ConnectionFactory::BuildLoginRequestCallback(),
    293       ConnectionHandler::ProtoReceivedCallback(),
    294       ConnectionHandler::ProtoSentCallback());
    295 }
    296 ConnectionFactoryImplTest::~ConnectionFactoryImplTest() {}
    297 
    298 void ConnectionFactoryImplTest::WaitForConnections() {
    299   run_loop_->Run();
    300   run_loop_.reset(new base::RunLoop());
    301 }
    302 
    303 void ConnectionFactoryImplTest::ConnectionsComplete() {
    304   if (!run_loop_)
    305     return;
    306   run_loop_->Quit();
    307 }
    308 
    309 void ConnectionFactoryImplTest::OnConnected(
    310     const GURL& current_server,
    311     const net::IPEndPoint& ip_endpoint) {
    312   connected_server_ = current_server;
    313 }
    314 
    315 void ConnectionFactoryImplTest::OnDisconnected() {
    316   connected_server_ = GURL();
    317 }
    318 
    319 // Verify building a connection handler works.
    320 TEST_F(ConnectionFactoryImplTest, Initialize) {
    321   ConnectionHandler* handler = factory()->GetConnectionHandler();
    322   ASSERT_TRUE(handler);
    323   EXPECT_FALSE(factory()->IsEndpointReachable());
    324   EXPECT_FALSE(connected_server().is_valid());
    325 }
    326 
    327 // An initial successful connection should not result in backoff.
    328 TEST_F(ConnectionFactoryImplTest, ConnectSuccess) {
    329   factory()->SetConnectResult(net::OK);
    330   factory()->Connect();
    331   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
    332   EXPECT_EQ(factory()->GetCurrentEndpoint(), BuildEndpoints()[0]);
    333   EXPECT_TRUE(factory()->IsEndpointReachable());
    334   EXPECT_TRUE(connected_server().is_valid());
    335 }
    336 
    337 // A connection failure should result in backoff, and attempting the fallback
    338 // endpoint next.
    339 TEST_F(ConnectionFactoryImplTest, ConnectFail) {
    340   factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
    341   factory()->Connect();
    342   EXPECT_FALSE(factory()->NextRetryAttempt().is_null());
    343   EXPECT_EQ(factory()->GetCurrentEndpoint(), BuildEndpoints()[1]);
    344   EXPECT_FALSE(factory()->IsEndpointReachable());
    345   EXPECT_FALSE(connected_server().is_valid());
    346 }
    347 
    348 // A connection success after a failure should reset backoff.
    349 TEST_F(ConnectionFactoryImplTest, FailThenSucceed) {
    350   factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
    351   base::TimeTicks connect_time = factory()->tick_clock()->NowTicks();
    352   factory()->Connect();
    353   WaitForConnections();
    354   EXPECT_FALSE(factory()->IsEndpointReachable());
    355   EXPECT_FALSE(connected_server().is_valid());
    356   base::TimeTicks retry_time = factory()->NextRetryAttempt();
    357   EXPECT_FALSE(retry_time.is_null());
    358   EXPECT_GE((retry_time - connect_time).InMilliseconds(), CalculateBackoff(1));
    359   factory()->SetConnectResult(net::OK);
    360   WaitForConnections();
    361   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
    362   EXPECT_TRUE(factory()->IsEndpointReachable());
    363   EXPECT_TRUE(connected_server().is_valid());
    364 }
    365 
    366 // Multiple connection failures should retry with an exponentially increasing
    367 // backoff, then reset on success.
    368 TEST_F(ConnectionFactoryImplTest, MultipleFailuresThenSucceed) {
    369   const int kNumAttempts = 5;
    370   factory()->SetMultipleConnectResults(net::ERR_CONNECTION_FAILED,
    371                                        kNumAttempts);
    372 
    373   base::TimeTicks connect_time = factory()->tick_clock()->NowTicks();
    374   factory()->Connect();
    375   WaitForConnections();
    376   EXPECT_FALSE(factory()->IsEndpointReachable());
    377   EXPECT_FALSE(connected_server().is_valid());
    378   base::TimeTicks retry_time = factory()->NextRetryAttempt();
    379   EXPECT_FALSE(retry_time.is_null());
    380   EXPECT_GE((retry_time - connect_time).InMilliseconds(),
    381             CalculateBackoff(kNumAttempts));
    382 
    383   factory()->SetConnectResult(net::OK);
    384   WaitForConnections();
    385   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
    386   EXPECT_TRUE(factory()->IsEndpointReachable());
    387   EXPECT_TRUE(connected_server().is_valid());
    388 }
    389 
    390 // Network change events should trigger canary connections.
    391 TEST_F(ConnectionFactoryImplTest, FailThenNetworkChangeEvent) {
    392   factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
    393   factory()->Connect();
    394   WaitForConnections();
    395   base::TimeTicks initial_backoff = factory()->NextRetryAttempt();
    396   EXPECT_FALSE(initial_backoff.is_null());
    397 
    398   factory()->SetConnectResult(net::ERR_FAILED);
    399   factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_WIFI);
    400   WaitForConnections();
    401 
    402   // Backoff should increase.
    403   base::TimeTicks next_backoff = factory()->NextRetryAttempt();
    404   EXPECT_GT(next_backoff, initial_backoff);
    405   EXPECT_FALSE(factory()->IsEndpointReachable());
    406 }
    407 
    408 // Verify that we reconnect even if a canary succeeded then disconnected while
    409 // a backoff was pending.
    410 TEST_F(ConnectionFactoryImplTest, CanarySucceedsThenDisconnects) {
    411   factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
    412   factory()->Connect();
    413   WaitForConnections();
    414   base::TimeTicks initial_backoff = factory()->NextRetryAttempt();
    415   EXPECT_FALSE(initial_backoff.is_null());
    416 
    417   factory()->SetConnectResult(net::OK);
    418   factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_ETHERNET);
    419   WaitForConnections();
    420   EXPECT_TRUE(factory()->IsEndpointReachable());
    421   EXPECT_TRUE(connected_server().is_valid());
    422 
    423   factory()->SetConnectResult(net::OK);
    424   factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
    425   EXPECT_FALSE(factory()->IsEndpointReachable());
    426   EXPECT_FALSE(connected_server().is_valid());
    427   WaitForConnections();
    428   EXPECT_TRUE(factory()->IsEndpointReachable());
    429   EXPECT_TRUE(connected_server().is_valid());
    430 }
    431 
    432 // Verify that if a canary connects, but hasn't finished the handshake, a
    433 // pending backoff attempt doesn't interrupt the connection.
    434 TEST_F(ConnectionFactoryImplTest, CanarySucceedsRetryDuringLogin) {
    435   factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
    436   factory()->Connect();
    437   WaitForConnections();
    438   base::TimeTicks initial_backoff = factory()->NextRetryAttempt();
    439   EXPECT_FALSE(initial_backoff.is_null());
    440 
    441   factory()->SetDelayLogin(true);
    442   factory()->SetConnectResult(net::OK);
    443   factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_WIFI);
    444   WaitForConnections();
    445   EXPECT_FALSE(factory()->IsEndpointReachable());
    446 
    447   // Pump the loop, to ensure the pending backoff retry has no effect.
    448   base::MessageLoop::current()->PostDelayedTask(
    449       FROM_HERE,
    450       base::MessageLoop::QuitClosure(),
    451       base::TimeDelta::FromMilliseconds(1));
    452   WaitForConnections();
    453 }
    454 
    455 // Fail after successful connection via signal reset.
    456 TEST_F(ConnectionFactoryImplTest, FailViaSignalReset) {
    457   factory()->SetConnectResult(net::OK);
    458   factory()->Connect();
    459   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
    460 
    461   factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
    462   EXPECT_FALSE(factory()->NextRetryAttempt().is_null());
    463   EXPECT_FALSE(factory()->IsEndpointReachable());
    464 }
    465 
    466 TEST_F(ConnectionFactoryImplTest, IgnoreResetWhileConnecting) {
    467   factory()->SetConnectResult(net::OK);
    468   factory()->Connect();
    469   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
    470 
    471   factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
    472   base::TimeTicks retry_time = factory()->NextRetryAttempt();
    473   EXPECT_FALSE(retry_time.is_null());
    474   EXPECT_FALSE(factory()->IsEndpointReachable());
    475 
    476   const int kNumAttempts = 5;
    477   for (int i = 0; i < kNumAttempts; ++i)
    478     factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
    479   EXPECT_EQ(retry_time, factory()->NextRetryAttempt());
    480   EXPECT_FALSE(factory()->IsEndpointReachable());
    481 }
    482 
    483 // Go into backoff due to connection failure. On successful connection, receive
    484 // a signal reset. The original backoff should be restored and extended, rather
    485 // than a new backoff starting from scratch.
    486 TEST_F(ConnectionFactoryImplTest, SignalResetRestoresBackoff) {
    487   factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
    488   base::TimeTicks connect_time = factory()->tick_clock()->NowTicks();
    489   factory()->Connect();
    490   WaitForConnections();
    491   base::TimeTicks retry_time = factory()->NextRetryAttempt();
    492   EXPECT_FALSE(retry_time.is_null());
    493 
    494   factory()->SetConnectResult(net::OK);
    495   connect_time = factory()->tick_clock()->NowTicks();
    496   WaitForConnections();
    497   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
    498 
    499   factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
    500   EXPECT_FALSE(factory()->IsEndpointReachable());
    501   EXPECT_FALSE(connected_server().is_valid());
    502   EXPECT_NE(retry_time, factory()->NextRetryAttempt());
    503   retry_time = factory()->NextRetryAttempt();
    504   EXPECT_FALSE(retry_time.is_null());
    505   EXPECT_GE((retry_time - connect_time).InMilliseconds(),
    506             CalculateBackoff(2));
    507 
    508   factory()->SetConnectResult(net::OK);
    509   connect_time = factory()->tick_clock()->NowTicks();
    510   factory()->tick_clock()->Advance(
    511       factory()->NextRetryAttempt() - connect_time);
    512   WaitForConnections();
    513   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
    514   EXPECT_TRUE(factory()->IsEndpointReachable());
    515   EXPECT_TRUE(connected_server().is_valid());
    516 
    517   factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
    518   EXPECT_NE(retry_time, factory()->NextRetryAttempt());
    519   retry_time = factory()->NextRetryAttempt();
    520   EXPECT_FALSE(retry_time.is_null());
    521   EXPECT_GE((retry_time - connect_time).InMilliseconds(),
    522             CalculateBackoff(3));
    523   EXPECT_FALSE(factory()->IsEndpointReachable());
    524   EXPECT_FALSE(connected_server().is_valid());
    525 }
    526 
    527 // When the network is disconnected, close the socket and suppress further
    528 // connection attempts until the network returns.
    529 // Disabled while crbug.com/396687 is being investigated.
    530 TEST_F(ConnectionFactoryImplTest, DISABLED_SuppressConnectWhenNoNetwork) {
    531   factory()->SetConnectResult(net::OK);
    532   factory()->Connect();
    533   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
    534   EXPECT_TRUE(factory()->IsEndpointReachable());
    535 
    536   // Advance clock so the login window reset isn't encountered.
    537   factory()->tick_clock()->Advance(base::TimeDelta::FromSeconds(11));
    538 
    539   // Will trigger reset, but will not attempt a new connection.
    540   factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_NONE);
    541   EXPECT_FALSE(factory()->IsEndpointReachable());
    542   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
    543 
    544   // When the network returns, attempt to connect.
    545   factory()->SetConnectResult(net::OK);
    546   factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_4G);
    547   WaitForConnections();
    548 
    549   EXPECT_TRUE(factory()->IsEndpointReachable());
    550   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
    551 }
    552 
    553 }  // namespace gcm
    554