Home | History | Annotate | Download | only in engine 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "base/bind.h"
      6 #include "base/callback.h"
      7 #include "base/compiler_specific.h"
      8 #include "base/memory/weak_ptr.h"
      9 #include "base/message_loop/message_loop.h"
     10 #include "base/test/test_timeouts.h"
     11 #include "sync/engine/backoff_delay_provider.h"
     12 #include "sync/engine/sync_scheduler_impl.h"
     13 #include "sync/engine/syncer.h"
     14 #include "sync/sessions/test_util.h"
     15 #include "sync/test/callback_counter.h"
     16 #include "sync/test/engine/fake_model_worker.h"
     17 #include "sync/test/engine/mock_connection_manager.h"
     18 #include "sync/test/engine/test_directory_setter_upper.h"
     19 #include "sync/util/extensions_activity.h"
     20 #include "testing/gmock/include/gmock/gmock.h"
     21 #include "testing/gtest/include/gtest/gtest.h"
     22 
     23 using base::TimeDelta;
     24 using base::TimeTicks;
     25 using testing::_;
     26 using testing::AtLeast;
     27 using testing::DoAll;
     28 using testing::Invoke;
     29 using testing::Mock;
     30 using testing::Return;
     31 using testing::WithArg;
     32 using testing::WithArgs;
     33 
     34 namespace syncer {
     35 using sessions::SyncSession;
     36 using sessions::SyncSessionContext;
     37 using sync_pb::GetUpdatesCallerInfo;
     38 
     39 class MockSyncer : public Syncer {
     40  public:
     41   MOCK_METHOD3(NormalSyncShare, bool(ModelTypeSet,
     42                                      const sessions::NudgeTracker&,
     43                                      sessions::SyncSession*));
     44   MOCK_METHOD3(ConfigureSyncShare,
     45                bool(ModelTypeSet,
     46                     sync_pb::GetUpdatesCallerInfo::GetUpdatesSource,
     47                     SyncSession*));
     48   MOCK_METHOD2(PollSyncShare, bool(ModelTypeSet, sessions::SyncSession*));
     49 };
     50 
     51 typedef std::vector<TimeTicks> SyncShareTimes;
     52 
     53 void QuitLoopNow() {
     54   // We use QuitNow() instead of Quit() as the latter may get stalled
     55   // indefinitely in the presence of repeated timers with low delays
     56   // and a slow test (e.g., ThrottlingDoesThrottle [which has a poll
     57   // delay of 5ms] run under TSAN on the trybots).
     58   base::MessageLoop::current()->QuitNow();
     59 }
     60 
     61 void RunLoop() {
     62   base::MessageLoop::current()->Run();
     63 }
     64 
     65 void PumpLoop() {
     66   // Do it this way instead of RunAllPending to pump loop exactly once
     67   // (necessary in the presence of timers; see comment in
     68   // QuitLoopNow).
     69   base::MessageLoop::current()->PostTask(FROM_HERE, base::Bind(&QuitLoopNow));
     70   RunLoop();
     71 }
     72 
     73 void PumpLoopFor(base::TimeDelta time) {
     74   // Allow the loop to run for the specified amount of time.
     75   base::MessageLoop::current()->PostDelayedTask(
     76       FROM_HERE, base::Bind(&QuitLoopNow), time);
     77   RunLoop();
     78 }
     79 
     80 ModelSafeRoutingInfo TypesToRoutingInfo(ModelTypeSet types) {
     81   ModelSafeRoutingInfo routes;
     82   for (ModelTypeSet::Iterator iter = types.First(); iter.Good(); iter.Inc()) {
     83     routes[iter.Get()] = GROUP_PASSIVE;
     84   }
     85   return routes;
     86 }
     87 
     88 // Convenient to use in tests wishing to analyze SyncShare calls over time.
     89 static const size_t kMinNumSamples = 5;
     90 class SyncSchedulerTest : public testing::Test {
     91  public:
     92   SyncSchedulerTest() : weak_ptr_factory_(this), syncer_(NULL), delay_(NULL) {}
     93 
     94   class MockDelayProvider : public BackoffDelayProvider {
     95    public:
     96     MockDelayProvider() : BackoffDelayProvider(
     97         TimeDelta::FromSeconds(kInitialBackoffRetrySeconds),
     98         TimeDelta::FromSeconds(kInitialBackoffImmediateRetrySeconds)) {
     99     }
    100 
    101     MOCK_METHOD1(GetDelay, TimeDelta(const TimeDelta&));
    102   };
    103 
    104   virtual void SetUp() {
    105     dir_maker_.SetUp();
    106     syncer_ = new testing::StrictMock<MockSyncer>();
    107     delay_ = NULL;
    108     extensions_activity_ = new ExtensionsActivity();
    109 
    110     routing_info_[BOOKMARKS] = GROUP_UI;
    111     routing_info_[AUTOFILL] = GROUP_DB;
    112     routing_info_[THEMES] = GROUP_UI;
    113     routing_info_[NIGORI] = GROUP_PASSIVE;
    114 
    115     workers_.clear();
    116     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_UI)));
    117     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_DB)));
    118     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_PASSIVE)));
    119 
    120     std::vector<ModelSafeWorker*> workers;
    121     for (std::vector<scoped_refptr<FakeModelWorker> >::iterator it =
    122          workers_.begin(); it != workers_.end(); ++it) {
    123       workers.push_back(it->get());
    124     }
    125 
    126     connection_.reset(new MockConnectionManager(directory()));
    127     connection_->SetServerReachable();
    128     context_.reset(new SyncSessionContext(
    129             connection_.get(), directory(), workers,
    130             extensions_activity_.get(),
    131             std::vector<SyncEngineEventListener*>(), NULL, NULL,
    132             true,  // enable keystore encryption
    133             false,  // force enable pre-commit GU avoidance
    134             "fake_invalidator_client_id"));
    135     context_->set_routing_info(routing_info_);
    136     context_->set_notifications_enabled(true);
    137     context_->set_account_name("Test");
    138     scheduler_.reset(
    139         new SyncSchedulerImpl("TestSyncScheduler",
    140             BackoffDelayProvider::FromDefaults(),
    141             context(),
    142             syncer_));
    143   }
    144 
    145   SyncSchedulerImpl* scheduler() { return scheduler_.get(); }
    146   const ModelSafeRoutingInfo& routing_info() { return routing_info_; }
    147   MockSyncer* syncer() { return syncer_; }
    148   MockDelayProvider* delay() { return delay_; }
    149   MockConnectionManager* connection() { return connection_.get(); }
    150   TimeDelta zero() { return TimeDelta::FromSeconds(0); }
    151   TimeDelta timeout() {
    152     return TestTimeouts::action_timeout();
    153   }
    154 
    155   virtual void TearDown() {
    156     PumpLoop();
    157     scheduler_.reset();
    158     PumpLoop();
    159     dir_maker_.TearDown();
    160   }
    161 
    162   void AnalyzePollRun(const SyncShareTimes& times, size_t min_num_samples,
    163       const TimeTicks& optimal_start, const TimeDelta& poll_interval) {
    164     EXPECT_GE(times.size(), min_num_samples);
    165     for (size_t i = 0; i < times.size(); i++) {
    166       SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
    167       TimeTicks optimal_next_sync = optimal_start + poll_interval * i;
    168       EXPECT_GE(times[i], optimal_next_sync);
    169     }
    170   }
    171 
    172   void DoQuitLoopNow() {
    173     QuitLoopNow();
    174   }
    175 
    176   void StartSyncScheduler(SyncScheduler::Mode mode) {
    177     scheduler()->Start(mode);
    178   }
    179 
    180   // This stops the scheduler synchronously.
    181   void StopSyncScheduler() {
    182     base::MessageLoop::current()->PostTask(
    183         FROM_HERE,
    184         base::Bind(&SyncSchedulerTest::DoQuitLoopNow,
    185                    weak_ptr_factory_.GetWeakPtr()));
    186     RunLoop();
    187   }
    188 
    189   bool RunAndGetBackoff() {
    190     ModelTypeSet nudge_types(BOOKMARKS);
    191     StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    192 
    193     scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
    194     RunLoop();
    195 
    196     return scheduler()->IsBackingOff();
    197   }
    198 
    199   void UseMockDelayProvider() {
    200     delay_ = new MockDelayProvider();
    201     scheduler_->delay_provider_.reset(delay_);
    202   }
    203 
    204   SyncSessionContext* context() { return context_.get(); }
    205 
    206   ModelTypeSet GetThrottledTypes() {
    207     return scheduler_->nudge_tracker_.GetThrottledTypes();
    208   }
    209 
    210  private:
    211   syncable::Directory* directory() {
    212     return dir_maker_.directory();
    213   }
    214 
    215   base::MessageLoop loop_;
    216   base::WeakPtrFactory<SyncSchedulerTest> weak_ptr_factory_;
    217   TestDirectorySetterUpper dir_maker_;
    218   scoped_ptr<MockConnectionManager> connection_;
    219   scoped_ptr<SyncSessionContext> context_;
    220   scoped_ptr<SyncSchedulerImpl> scheduler_;
    221   MockSyncer* syncer_;
    222   MockDelayProvider* delay_;
    223   std::vector<scoped_refptr<FakeModelWorker> > workers_;
    224   scoped_refptr<ExtensionsActivity> extensions_activity_;
    225   ModelSafeRoutingInfo routing_info_;
    226 };
    227 
    228 void RecordSyncShareImpl(SyncShareTimes* times) {
    229   times->push_back(TimeTicks::Now());
    230 }
    231 
    232 ACTION_P(RecordSyncShare, times) {
    233   RecordSyncShareImpl(times);
    234   if (base::MessageLoop::current()->is_running())
    235     QuitLoopNow();
    236   return true;
    237 }
    238 
    239 ACTION_P2(RecordSyncShareMultiple, times, quit_after) {
    240   RecordSyncShareImpl(times);
    241   EXPECT_LE(times->size(), quit_after);
    242   if (times->size() >= quit_after &&
    243       base::MessageLoop::current()->is_running()) {
    244     QuitLoopNow();
    245   }
    246   return true;
    247 }
    248 
    249 ACTION(AddFailureAndQuitLoopNow) {
    250   ADD_FAILURE();
    251   QuitLoopNow();
    252   return true;
    253 }
    254 
    255 ACTION(QuitLoopNowAction) {
    256   QuitLoopNow();
    257   return true;
    258 }
    259 
    260 // Test nudge scheduling.
    261 TEST_F(SyncSchedulerTest, Nudge) {
    262   SyncShareTimes times;
    263   ModelTypeSet model_types(BOOKMARKS);
    264 
    265   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    266       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    267                       RecordSyncShare(&times)))
    268       .RetiresOnSaturation();
    269 
    270   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    271 
    272   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
    273   RunLoop();
    274 
    275   Mock::VerifyAndClearExpectations(syncer());
    276 
    277   // Make sure a second, later, nudge is unaffected by first (no coalescing).
    278   SyncShareTimes times2;
    279   model_types.Remove(BOOKMARKS);
    280   model_types.Put(AUTOFILL);
    281   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    282       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    283                       RecordSyncShare(&times2)));
    284   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
    285   RunLoop();
    286 }
    287 
    288 // Make sure a regular config command is scheduled fine in the absence of any
    289 // errors.
    290 TEST_F(SyncSchedulerTest, Config) {
    291   SyncShareTimes times;
    292   const ModelTypeSet model_types(BOOKMARKS);
    293 
    294   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    295       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
    296                       RecordSyncShare(&times)));
    297 
    298   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    299 
    300   CallbackCounter counter;
    301   ConfigurationParams params(
    302       GetUpdatesCallerInfo::RECONFIGURATION,
    303       model_types,
    304       TypesToRoutingInfo(model_types),
    305       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
    306   ASSERT_TRUE(scheduler()->ScheduleConfiguration(params));
    307   ASSERT_EQ(1, counter.times_called());
    308 }
    309 
    310 // Simulate a failure and make sure the config request is retried.
    311 TEST_F(SyncSchedulerTest, ConfigWithBackingOff) {
    312   UseMockDelayProvider();
    313   EXPECT_CALL(*delay(), GetDelay(_))
    314       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
    315   SyncShareTimes times;
    316   const ModelTypeSet model_types(BOOKMARKS);
    317 
    318   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    319 
    320   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    321       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
    322                       RecordSyncShare(&times)));
    323 
    324   CallbackCounter counter;
    325   ConfigurationParams params(
    326       GetUpdatesCallerInfo::RECONFIGURATION,
    327       model_types,
    328       TypesToRoutingInfo(model_types),
    329       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
    330   ASSERT_FALSE(scheduler()->ScheduleConfiguration(params));
    331   ASSERT_EQ(0, counter.times_called());
    332 
    333   Mock::VerifyAndClearExpectations(syncer());
    334 
    335   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    336       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
    337                       RecordSyncShare(&times)));
    338   RunLoop();
    339 
    340   ASSERT_EQ(1, counter.times_called());
    341 }
    342 
    343 // Issue a nudge when the config has failed. Make sure both the config and
    344 // nudge are executed.
    345 TEST_F(SyncSchedulerTest, NudgeWithConfigWithBackingOff) {
    346   const ModelTypeSet model_types(BOOKMARKS);
    347   UseMockDelayProvider();
    348   EXPECT_CALL(*delay(), GetDelay(_))
    349       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(50)));
    350   SyncShareTimes times;
    351 
    352   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    353 
    354   // Request a configure and make sure it fails.
    355   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    356       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
    357                       RecordSyncShare(&times)));
    358   CallbackCounter counter;
    359   ConfigurationParams params(
    360       GetUpdatesCallerInfo::RECONFIGURATION,
    361       model_types,
    362       TypesToRoutingInfo(model_types),
    363       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
    364   ASSERT_FALSE(scheduler()->ScheduleConfiguration(params));
    365   ASSERT_EQ(0, counter.times_called());
    366   Mock::VerifyAndClearExpectations(syncer());
    367 
    368   // Ask for a nudge while dealing with repeated configure failure.
    369   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    370       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
    371                       RecordSyncShare(&times)));
    372   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
    373   RunLoop();
    374   // Note that we're not RunLoop()ing for the NUDGE we just scheduled, but
    375   // for the first retry attempt from the config job (after
    376   // waiting ~+/- 50ms).
    377   Mock::VerifyAndClearExpectations(syncer());
    378   ASSERT_EQ(0, counter.times_called());
    379 
    380   // Let the next configure retry succeed.
    381   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    382       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
    383                       RecordSyncShare(&times)));
    384   RunLoop();
    385 
    386   // Now change the mode so nudge can execute.
    387   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    388       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    389                       RecordSyncShare(&times)));
    390   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    391 }
    392 
    393 // Test that nudges are coalesced.
    394 TEST_F(SyncSchedulerTest, NudgeCoalescing) {
    395   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    396 
    397   SyncShareTimes times;
    398   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    399       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    400                       RecordSyncShare(&times)));
    401   const ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3(THEMES);
    402   TimeDelta delay = zero();
    403   TimeTicks optimal_time = TimeTicks::Now() + delay;
    404   scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
    405   scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
    406   RunLoop();
    407 
    408   ASSERT_EQ(1U, times.size());
    409   EXPECT_GE(times[0], optimal_time);
    410 
    411   Mock::VerifyAndClearExpectations(syncer());
    412 
    413   SyncShareTimes times2;
    414   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    415       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    416                       RecordSyncShare(&times2)));
    417   scheduler()->ScheduleLocalNudge(zero(), types3, FROM_HERE);
    418   RunLoop();
    419 }
    420 
    421 // Test that nudges are coalesced.
    422 TEST_F(SyncSchedulerTest, NudgeCoalescingWithDifferentTimings) {
    423   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    424 
    425   SyncShareTimes times;
    426   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    427       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    428                       RecordSyncShare(&times)));
    429   ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3;
    430 
    431   // Create a huge time delay.
    432   TimeDelta delay = TimeDelta::FromDays(1);
    433 
    434   scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
    435   scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
    436 
    437   TimeTicks min_time = TimeTicks::Now();
    438   TimeTicks max_time = TimeTicks::Now() + delay;
    439 
    440   RunLoop();
    441   Mock::VerifyAndClearExpectations(syncer());
    442 
    443   // Make sure the sync happened at the right time.
    444   ASSERT_EQ(1U, times.size());
    445   EXPECT_GE(times[0], min_time);
    446   EXPECT_LE(times[0], max_time);
    447 }
    448 
    449 // Test nudge scheduling.
    450 TEST_F(SyncSchedulerTest, NudgeWithStates) {
    451   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    452 
    453   SyncShareTimes times;
    454   const ModelTypeSet types(BOOKMARKS);
    455   ModelTypeInvalidationMap invalidation_map =
    456       ModelTypeSetToInvalidationMap(types, "test");
    457 
    458   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    459       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    460                       RecordSyncShare(&times)))
    461       .RetiresOnSaturation();
    462   scheduler()->ScheduleInvalidationNudge(zero(), invalidation_map, FROM_HERE);
    463   RunLoop();
    464 
    465   Mock::VerifyAndClearExpectations(syncer());
    466 
    467   // Make sure a second, later, nudge is unaffected by first (no coalescing).
    468   SyncShareTimes times2;
    469   invalidation_map.erase(BOOKMARKS);
    470   invalidation_map[AUTOFILL].payload = "test2";
    471   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    472       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    473                       RecordSyncShare(&times2)));
    474   scheduler()->ScheduleInvalidationNudge(zero(), invalidation_map, FROM_HERE);
    475   RunLoop();
    476 }
    477 
    478 // Test that polling works as expected.
    479 TEST_F(SyncSchedulerTest, Polling) {
    480   SyncShareTimes times;
    481   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
    482   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
    483       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
    484            RecordSyncShareMultiple(&times, kMinNumSamples)));
    485 
    486   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
    487 
    488   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
    489   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    490 
    491   // Run again to wait for polling.
    492   RunLoop();
    493 
    494   StopSyncScheduler();
    495   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
    496 }
    497 
    498 // Test that the short poll interval is used.
    499 TEST_F(SyncSchedulerTest, PollNotificationsDisabled) {
    500   SyncShareTimes times;
    501   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
    502   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
    503       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
    504            RecordSyncShareMultiple(&times, kMinNumSamples)));
    505 
    506   scheduler()->OnReceivedShortPollIntervalUpdate(poll_interval);
    507   scheduler()->SetNotificationsEnabled(false);
    508 
    509   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
    510   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    511 
    512   // Run again to wait for polling.
    513   RunLoop();
    514 
    515   StopSyncScheduler();
    516   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
    517 }
    518 
    519 // Test that polling intervals are updated when needed.
    520 TEST_F(SyncSchedulerTest, PollIntervalUpdate) {
    521   SyncShareTimes times;
    522   TimeDelta poll1(TimeDelta::FromMilliseconds(120));
    523   TimeDelta poll2(TimeDelta::FromMilliseconds(30));
    524   scheduler()->OnReceivedLongPollIntervalUpdate(poll1);
    525   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
    526       .WillOnce(DoAll(
    527           WithArgs<0,1>(
    528               sessions::test_util::SimulatePollIntervalUpdate(poll2)),
    529           Return(true)))
    530       .WillRepeatedly(
    531           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
    532                 WithArg<1>(
    533                     RecordSyncShareMultiple(&times, kMinNumSamples))));
    534 
    535   TimeTicks optimal_start = TimeTicks::Now() + poll1 + poll2;
    536   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    537 
    538   // Run again to wait for polling.
    539   RunLoop();
    540 
    541   StopSyncScheduler();
    542   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll2);
    543 }
    544 
    545 // Test that the sessions commit delay is updated when needed.
    546 TEST_F(SyncSchedulerTest, SessionsCommitDelay) {
    547   SyncShareTimes times;
    548   TimeDelta delay1(TimeDelta::FromMilliseconds(120));
    549   TimeDelta delay2(TimeDelta::FromMilliseconds(30));
    550   scheduler()->OnReceivedSessionsCommitDelay(delay1);
    551 
    552   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    553       .WillOnce(
    554           DoAll(
    555               WithArgs<0,1,2>(
    556                   sessions::test_util::SimulateSessionsCommitDelayUpdate(
    557                       delay2)),
    558               Invoke(sessions::test_util::SimulateNormalSuccess),
    559               QuitLoopNowAction()));
    560 
    561   EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
    562   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    563 
    564   EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
    565   const ModelTypeSet model_types(BOOKMARKS);
    566   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
    567   RunLoop();
    568 
    569   EXPECT_EQ(delay2, scheduler()->GetSessionsCommitDelay());
    570   StopSyncScheduler();
    571 }
    572 
    573 // Test that no syncing occurs when throttled.
    574 TEST_F(SyncSchedulerTest, ThrottlingDoesThrottle) {
    575   const ModelTypeSet types(BOOKMARKS);
    576   TimeDelta poll(TimeDelta::FromMilliseconds(5));
    577   TimeDelta throttle(TimeDelta::FromMinutes(10));
    578   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    579 
    580   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    581       .WillOnce(DoAll(
    582           WithArg<2>(sessions::test_util::SimulateThrottled(throttle)),
    583           Return(true)))
    584       .WillRepeatedly(AddFailureAndQuitLoopNow());
    585 
    586   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    587 
    588   scheduler()->ScheduleLocalNudge(
    589       TimeDelta::FromMicroseconds(1), types, FROM_HERE);
    590   PumpLoop();
    591 
    592   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    593 
    594   CallbackCounter counter;
    595   ConfigurationParams params(
    596       GetUpdatesCallerInfo::RECONFIGURATION,
    597       types,
    598       TypesToRoutingInfo(types),
    599       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
    600   ASSERT_FALSE(scheduler()->ScheduleConfiguration(params));
    601   ASSERT_EQ(0, counter.times_called());
    602 }
    603 
    604 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromPoll) {
    605   SyncShareTimes times;
    606   TimeDelta poll(TimeDelta::FromMilliseconds(15));
    607   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
    608   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    609 
    610   ::testing::InSequence seq;
    611   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
    612       .WillOnce(DoAll(
    613           WithArg<1>(sessions::test_util::SimulateThrottled(throttle1)),
    614           Return(true)))
    615       .RetiresOnSaturation();
    616   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
    617       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
    618            RecordSyncShareMultiple(&times, kMinNumSamples)));
    619 
    620   TimeTicks optimal_start = TimeTicks::Now() + poll + throttle1;
    621   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    622 
    623   // Run again to wait for polling.
    624   RunLoop();
    625 
    626   StopSyncScheduler();
    627   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll);
    628 }
    629 
    630 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromNudge) {
    631   SyncShareTimes times;
    632   TimeDelta poll(TimeDelta::FromDays(1));
    633   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
    634   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    635 
    636   ::testing::InSequence seq;
    637   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    638       .WillOnce(DoAll(
    639           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
    640           Return(true)))
    641       .RetiresOnSaturation();
    642   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    643       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    644                       QuitLoopNowAction()));
    645 
    646   const ModelTypeSet types(BOOKMARKS);
    647   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    648   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
    649 
    650   PumpLoop();
    651   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
    652   RunLoop();
    653   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
    654 
    655   StopSyncScheduler();
    656 }
    657 
    658 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromConfigure) {
    659   SyncShareTimes times;
    660   TimeDelta poll(TimeDelta::FromDays(1));
    661   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
    662   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    663 
    664   ::testing::InSequence seq;
    665   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    666       .WillOnce(DoAll(
    667           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
    668           Return(true)))
    669       .RetiresOnSaturation();
    670   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    671       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
    672                       QuitLoopNowAction()));
    673 
    674   const ModelTypeSet types(BOOKMARKS);
    675   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    676 
    677   CallbackCounter counter;
    678   ConfigurationParams params(
    679       GetUpdatesCallerInfo::RECONFIGURATION,
    680       types,
    681       TypesToRoutingInfo(types),
    682       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
    683   EXPECT_FALSE(scheduler()->ScheduleConfiguration(params));
    684   EXPECT_EQ(0, counter.times_called());
    685   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
    686 
    687   RunLoop();
    688   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
    689 
    690   StopSyncScheduler();
    691 }
    692 
    693 TEST_F(SyncSchedulerTest, TypeThrottlingBlocksNudge) {
    694   UseMockDelayProvider();
    695   EXPECT_CALL(*delay(), GetDelay(_))
    696       .WillRepeatedly(Return(zero()));
    697 
    698   TimeDelta poll(TimeDelta::FromDays(1));
    699   TimeDelta throttle1(TimeDelta::FromSeconds(60));
    700   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    701 
    702   const ModelTypeSet types(BOOKMARKS);
    703 
    704   ::testing::InSequence seq;
    705   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    706       .WillOnce(DoAll(
    707           WithArg<2>(
    708               sessions::test_util::SimulateTypesThrottled(types, throttle1)),
    709           Return(true)))
    710       .RetiresOnSaturation();
    711 
    712   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    713   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
    714   PumpLoop();
    715   EXPECT_TRUE(GetThrottledTypes().HasAll(types));
    716 
    717   // This won't cause a sync cycle because the types are throttled.
    718   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
    719   PumpLoop();
    720 
    721   StopSyncScheduler();
    722 }
    723 
    724 TEST_F(SyncSchedulerTest, TypeThrottlingDoesBlockOtherSources) {
    725   UseMockDelayProvider();
    726   EXPECT_CALL(*delay(), GetDelay(_))
    727       .WillRepeatedly(Return(zero()));
    728 
    729   SyncShareTimes times;
    730   TimeDelta poll(TimeDelta::FromDays(1));
    731   TimeDelta throttle1(TimeDelta::FromSeconds(60));
    732   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    733 
    734   const ModelTypeSet throttled_types(BOOKMARKS);
    735   const ModelTypeSet unthrottled_types(PREFERENCES);
    736 
    737   ::testing::InSequence seq;
    738   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    739       .WillOnce(DoAll(
    740           WithArg<2>(
    741               sessions::test_util::SimulateTypesThrottled(
    742                   throttled_types, throttle1)),
    743           Return(true)))
    744       .RetiresOnSaturation();
    745 
    746   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    747   scheduler()->ScheduleLocalNudge(zero(), throttled_types, FROM_HERE);
    748   PumpLoop();
    749   EXPECT_TRUE(GetThrottledTypes().HasAll(throttled_types));
    750 
    751   // Ignore invalidations for throttled types.
    752   ModelTypeInvalidationMap invalidation_map =
    753       ModelTypeSetToInvalidationMap(throttled_types, "test");
    754   scheduler()->ScheduleInvalidationNudge(zero(), invalidation_map, FROM_HERE);
    755   PumpLoop();
    756 
    757   // Ignore refresh requests for throttled types.
    758   scheduler()->ScheduleLocalRefreshRequest(zero(), throttled_types, FROM_HERE);
    759   PumpLoop();
    760 
    761   Mock::VerifyAndClearExpectations(syncer());
    762 
    763   // Local nudges for non-throttled types will trigger a sync.
    764   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    765       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    766                             RecordSyncShare(&times)));
    767   scheduler()->ScheduleLocalNudge(zero(), unthrottled_types, FROM_HERE);
    768   RunLoop();
    769   Mock::VerifyAndClearExpectations(syncer());
    770 
    771   StopSyncScheduler();
    772 }
    773 
    774 // Test nudges / polls don't run in config mode and config tasks do.
    775 TEST_F(SyncSchedulerTest, ConfigurationMode) {
    776   TimeDelta poll(TimeDelta::FromMilliseconds(15));
    777   SyncShareTimes times;
    778   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    779 
    780   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    781 
    782   const ModelTypeSet nudge_types(AUTOFILL);
    783   scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
    784   scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
    785 
    786   const ModelTypeSet config_types(BOOKMARKS);
    787 
    788   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    789       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
    790                       RecordSyncShare(&times)))
    791       .RetiresOnSaturation();
    792   CallbackCounter counter;
    793   ConfigurationParams params(
    794       GetUpdatesCallerInfo::RECONFIGURATION,
    795       config_types,
    796       TypesToRoutingInfo(config_types),
    797       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
    798   ASSERT_TRUE(scheduler()->ScheduleConfiguration(params));
    799   ASSERT_EQ(1, counter.times_called());
    800   Mock::VerifyAndClearExpectations(syncer());
    801 
    802   // Switch to NORMAL_MODE to ensure NUDGES were properly saved and run.
    803   scheduler()->OnReceivedLongPollIntervalUpdate(TimeDelta::FromDays(1));
    804   SyncShareTimes times2;
    805   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    806       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    807                       RecordSyncShare(&times2)));
    808 
    809   // TODO(tim): Figure out how to remove this dangerous need to reset
    810   // routing info between mode switches.
    811   context()->set_routing_info(routing_info());
    812   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    813 
    814   PumpLoop();
    815 }
    816 
    817 class BackoffTriggersSyncSchedulerTest : public SyncSchedulerTest {
    818   virtual void SetUp() {
    819     SyncSchedulerTest::SetUp();
    820     UseMockDelayProvider();
    821     EXPECT_CALL(*delay(), GetDelay(_))
    822         .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
    823   }
    824 
    825   virtual void TearDown() {
    826     StopSyncScheduler();
    827     SyncSchedulerTest::TearDown();
    828   }
    829 };
    830 
    831 // Have the sycner fail during commit.  Expect that the scheduler enters
    832 // backoff.
    833 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnce) {
    834   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    835       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
    836                       QuitLoopNowAction()));
    837   EXPECT_TRUE(RunAndGetBackoff());
    838 }
    839 
    840 // Have the syncer fail during download updates and succeed on the first
    841 // retry.  Expect that this clears the backoff state.
    842 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadOnceThenSucceed) {
    843   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    844       .WillOnce(DoAll(
    845           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
    846           Return(true)))
    847       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    848                       QuitLoopNowAction()));
    849   EXPECT_FALSE(RunAndGetBackoff());
    850 }
    851 
    852 // Have the syncer fail during commit and succeed on the first retry.  Expect
    853 // that this clears the backoff state.
    854 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnceThenSucceed) {
    855   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    856       .WillOnce(DoAll(
    857           Invoke(sessions::test_util::SimulateCommitFailed),
    858           Return(true)))
    859       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    860                       QuitLoopNowAction()));
    861   EXPECT_FALSE(RunAndGetBackoff());
    862 }
    863 
    864 // Have the syncer fail to download updates and fail again on the retry.
    865 // Expect this will leave the scheduler in backoff.
    866 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadTwice) {
    867   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    868       .WillOnce(DoAll(
    869           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
    870           Return(true)))
    871       .WillRepeatedly(DoAll(
    872               Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
    873               QuitLoopNowAction()));
    874   EXPECT_TRUE(RunAndGetBackoff());
    875 }
    876 
    877 // Have the syncer fail to get the encryption key yet succeed in downloading
    878 // updates. Expect this will leave the scheduler in backoff.
    879 TEST_F(BackoffTriggersSyncSchedulerTest, FailGetEncryptionKey) {
    880   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    881       .WillOnce(DoAll(
    882           Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
    883           Return(true)))
    884       .WillRepeatedly(DoAll(
    885               Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
    886               QuitLoopNowAction()));
    887   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    888 
    889   ModelTypeSet types(BOOKMARKS);
    890   CallbackCounter counter;
    891   ConfigurationParams params(
    892       GetUpdatesCallerInfo::RECONFIGURATION,
    893       types,
    894       TypesToRoutingInfo(types),
    895       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
    896   scheduler()->ScheduleConfiguration(params);
    897   RunLoop();
    898 
    899   EXPECT_TRUE(scheduler()->IsBackingOff());
    900 }
    901 
    902 // Test that no polls or extraneous nudges occur when in backoff.
    903 TEST_F(SyncSchedulerTest, BackoffDropsJobs) {
    904   SyncShareTimes times;
    905   TimeDelta poll(TimeDelta::FromMilliseconds(5));
    906   const ModelTypeSet types(BOOKMARKS);
    907   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    908   UseMockDelayProvider();
    909 
    910   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    911       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
    912                       RecordSyncShareMultiple(&times, 1U)));
    913   EXPECT_CALL(*delay(), GetDelay(_)).
    914       WillRepeatedly(Return(TimeDelta::FromDays(1)));
    915 
    916   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    917 
    918   // This nudge should fail and put us into backoff.  Thanks to our mock
    919   // GetDelay() setup above, this will be a long backoff.
    920   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
    921   RunLoop();
    922 
    923   // From this point forward, no SyncShare functions should be invoked.
    924   Mock::VerifyAndClearExpectations(syncer());
    925 
    926   // Wait a while (10x poll interval) so a few poll jobs will be attempted.
    927   PumpLoopFor(poll * 10);
    928 
    929   // Try (and fail) to schedule a nudge.
    930   scheduler()->ScheduleLocalNudge(
    931       base::TimeDelta::FromMilliseconds(1),
    932       types,
    933       FROM_HERE);
    934 
    935   Mock::VerifyAndClearExpectations(syncer());
    936   Mock::VerifyAndClearExpectations(delay());
    937 
    938   EXPECT_CALL(*delay(), GetDelay(_)).Times(0);
    939 
    940   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    941 
    942   CallbackCounter counter;
    943   ConfigurationParams params(
    944       GetUpdatesCallerInfo::RECONFIGURATION,
    945       types,
    946       TypesToRoutingInfo(types),
    947       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
    948   ASSERT_FALSE(scheduler()->ScheduleConfiguration(params));
    949   ASSERT_EQ(0, counter.times_called());
    950 }
    951 
    952 // Test that backoff is shaping traffic properly with consecutive errors.
    953 TEST_F(SyncSchedulerTest, BackoffElevation) {
    954   SyncShareTimes times;
    955   UseMockDelayProvider();
    956 
    957   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_)).Times(kMinNumSamples)
    958       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
    959           RecordSyncShareMultiple(&times, kMinNumSamples)));
    960 
    961   const TimeDelta first = TimeDelta::FromSeconds(kInitialBackoffRetrySeconds);
    962   const TimeDelta second = TimeDelta::FromMilliseconds(2);
    963   const TimeDelta third = TimeDelta::FromMilliseconds(3);
    964   const TimeDelta fourth = TimeDelta::FromMilliseconds(4);
    965   const TimeDelta fifth = TimeDelta::FromMilliseconds(5);
    966   const TimeDelta sixth = TimeDelta::FromDays(1);
    967 
    968   EXPECT_CALL(*delay(), GetDelay(first)).WillOnce(Return(second))
    969           .RetiresOnSaturation();
    970   EXPECT_CALL(*delay(), GetDelay(second)).WillOnce(Return(third))
    971           .RetiresOnSaturation();
    972   EXPECT_CALL(*delay(), GetDelay(third)).WillOnce(Return(fourth))
    973           .RetiresOnSaturation();
    974   EXPECT_CALL(*delay(), GetDelay(fourth)).WillOnce(Return(fifth))
    975           .RetiresOnSaturation();
    976   EXPECT_CALL(*delay(), GetDelay(fifth)).WillOnce(Return(sixth));
    977 
    978   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    979 
    980   // Run again with a nudge.
    981   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
    982   RunLoop();
    983 
    984   ASSERT_EQ(kMinNumSamples, times.size());
    985   EXPECT_GE(times[1] - times[0], second);
    986   EXPECT_GE(times[2] - times[1], third);
    987   EXPECT_GE(times[3] - times[2], fourth);
    988   EXPECT_GE(times[4] - times[3], fifth);
    989 }
    990 
    991 // Test that things go back to normal once a retry makes forward progress.
    992 TEST_F(SyncSchedulerTest, BackoffRelief) {
    993   SyncShareTimes times;
    994   const TimeDelta poll(TimeDelta::FromMilliseconds(10));
    995   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    996   UseMockDelayProvider();
    997 
    998   const TimeDelta backoff = TimeDelta::FromMilliseconds(5);
    999   EXPECT_CALL(*delay(), GetDelay(_)).WillOnce(Return(backoff));
   1000 
   1001   // Optimal start for the post-backoff poll party.
   1002   TimeTicks optimal_start = TimeTicks::Now();
   1003   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1004 
   1005   // Kick off the test with a failed nudge.
   1006   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1007       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
   1008                       RecordSyncShare(&times)));
   1009   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
   1010   RunLoop();
   1011   Mock::VerifyAndClearExpectations(syncer());
   1012   TimeTicks optimal_job_time = optimal_start;
   1013   ASSERT_EQ(1U, times.size());
   1014   EXPECT_GE(times[0], optimal_job_time);
   1015 
   1016   // The retry succeeds.
   1017   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1018       .WillOnce(DoAll(
   1019               Invoke(sessions::test_util::SimulateNormalSuccess),
   1020               RecordSyncShare(&times)));
   1021   RunLoop();
   1022   Mock::VerifyAndClearExpectations(syncer());
   1023   optimal_job_time = optimal_job_time + backoff;
   1024   ASSERT_EQ(2U, times.size());
   1025   EXPECT_GE(times[1], optimal_job_time);
   1026 
   1027   // Now let the Poll timer do its thing.
   1028   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
   1029       .WillRepeatedly(DoAll(
   1030               Invoke(sessions::test_util::SimulatePollSuccess),
   1031               RecordSyncShareMultiple(&times, kMinNumSamples)));
   1032   RunLoop();
   1033   Mock::VerifyAndClearExpectations(syncer());
   1034   ASSERT_EQ(kMinNumSamples, times.size());
   1035   for (size_t i = 2; i < times.size(); i++) {
   1036     optimal_job_time = optimal_job_time + poll;
   1037     SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
   1038     EXPECT_GE(times[i], optimal_job_time);
   1039   }
   1040 
   1041   StopSyncScheduler();
   1042 }
   1043 
   1044 // Test that poll failures are ignored.  They should have no effect on
   1045 // subsequent poll attempts, nor should they trigger a backoff/retry.
   1046 TEST_F(SyncSchedulerTest, TransientPollFailure) {
   1047   SyncShareTimes times;
   1048   const TimeDelta poll_interval(TimeDelta::FromMilliseconds(1));
   1049   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
   1050   UseMockDelayProvider(); // Will cause test failure if backoff is initiated.
   1051 
   1052   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
   1053       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollFailed),
   1054                       RecordSyncShare(&times)))
   1055       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
   1056                       RecordSyncShare(&times)));
   1057 
   1058   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1059 
   1060   // Run the unsucessful poll. The failed poll should not trigger backoff.
   1061   RunLoop();
   1062   EXPECT_FALSE(scheduler()->IsBackingOff());
   1063 
   1064   // Run the successful poll.
   1065   RunLoop();
   1066   EXPECT_FALSE(scheduler()->IsBackingOff());
   1067 }
   1068 
   1069 // Test that starting the syncer thread without a valid connection doesn't
   1070 // break things when a connection is detected.
   1071 TEST_F(SyncSchedulerTest, StartWhenNotConnected) {
   1072   connection()->SetServerNotReachable();
   1073   connection()->UpdateConnectionStatus();
   1074   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1075     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
   1076                     Return(true)))
   1077     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
   1078                     Return(true)));
   1079   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1080 
   1081   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
   1082   // Should save the nudge for until after the server is reachable.
   1083   base::MessageLoop::current()->RunUntilIdle();
   1084 
   1085   scheduler()->OnConnectionStatusChange();
   1086   connection()->SetServerReachable();
   1087   connection()->UpdateConnectionStatus();
   1088   base::MessageLoop::current()->RunUntilIdle();
   1089 }
   1090 
   1091 TEST_F(SyncSchedulerTest, ServerConnectionChangeDuringBackoff) {
   1092   UseMockDelayProvider();
   1093   EXPECT_CALL(*delay(), GetDelay(_))
   1094       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
   1095 
   1096   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1097   connection()->SetServerNotReachable();
   1098   connection()->UpdateConnectionStatus();
   1099 
   1100   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1101     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
   1102                     Return(true)))
   1103     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
   1104                     Return(true)));
   1105 
   1106   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
   1107 
   1108   PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
   1109   ASSERT_TRUE(scheduler()->IsBackingOff());
   1110 
   1111   // Before we run the scheduled canary, trigger a server connection change.
   1112   scheduler()->OnConnectionStatusChange();
   1113   connection()->SetServerReachable();
   1114   connection()->UpdateConnectionStatus();
   1115   base::MessageLoop::current()->RunUntilIdle();
   1116 }
   1117 
   1118 // This was supposed to test the scenario where we receive a nudge while a
   1119 // connection change canary is scheduled, but has not run yet.  Since we've made
   1120 // the connection change canary synchronous, this is no longer possible.
   1121 TEST_F(SyncSchedulerTest, ConnectionChangeCanaryPreemptedByNudge) {
   1122   UseMockDelayProvider();
   1123   EXPECT_CALL(*delay(), GetDelay(_))
   1124       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
   1125 
   1126   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1127   connection()->SetServerNotReachable();
   1128   connection()->UpdateConnectionStatus();
   1129 
   1130   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1131     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
   1132                     Return(true)))
   1133     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
   1134                     Return(true)))
   1135     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
   1136                     QuitLoopNowAction()));
   1137 
   1138   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
   1139 
   1140   PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
   1141   ASSERT_TRUE(scheduler()->IsBackingOff());
   1142 
   1143   // Before we run the scheduled canary, trigger a server connection change.
   1144   scheduler()->OnConnectionStatusChange();
   1145   connection()->SetServerReachable();
   1146   connection()->UpdateConnectionStatus();
   1147   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
   1148   base::MessageLoop::current()->RunUntilIdle();
   1149 }
   1150 
   1151 // Tests that we don't crash trying to run two canaries at once if we receive
   1152 // extra connection status change notifications.  See crbug.com/190085.
   1153 TEST_F(SyncSchedulerTest, DoubleCanaryInConfigure) {
   1154   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
   1155       .WillRepeatedly(DoAll(
   1156               Invoke(sessions::test_util::SimulateConfigureConnectionFailure),
   1157               Return(true)));
   1158   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
   1159   connection()->SetServerNotReachable();
   1160   connection()->UpdateConnectionStatus();
   1161 
   1162   ModelTypeSet model_types(BOOKMARKS);
   1163   CallbackCounter counter;
   1164   ConfigurationParams params(
   1165       GetUpdatesCallerInfo::RECONFIGURATION,
   1166       model_types,
   1167       TypesToRoutingInfo(model_types),
   1168       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
   1169   scheduler()->ScheduleConfiguration(params);
   1170 
   1171   scheduler()->OnConnectionStatusChange();
   1172   scheduler()->OnConnectionStatusChange();
   1173 
   1174   PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
   1175 }
   1176 
   1177 TEST_F(SyncSchedulerTest, PollFromCanaryAfterAuthError) {
   1178   SyncShareTimes times;
   1179   TimeDelta poll(TimeDelta::FromMilliseconds(15));
   1180   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
   1181 
   1182   ::testing::InSequence seq;
   1183   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
   1184       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
   1185            RecordSyncShareMultiple(&times, kMinNumSamples)));
   1186 
   1187   connection()->SetServerStatus(HttpResponse::SYNC_AUTH_ERROR);
   1188   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1189 
   1190   // Run to wait for polling.
   1191   RunLoop();
   1192 
   1193   // Normally OnCredentialsUpdated calls TryCanaryJob that doesn't run Poll,
   1194   // but after poll finished with auth error from poll timer it should retry
   1195   // poll once more
   1196   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
   1197       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
   1198                       RecordSyncShare(&times)));
   1199   scheduler()->OnCredentialsUpdated();
   1200   connection()->SetServerStatus(HttpResponse::SERVER_CONNECTION_OK);
   1201   StopSyncScheduler();
   1202 }
   1203 
   1204 }  // namespace syncer
   1205