Home | History | Annotate | Download | only in engine 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "base/bind.h"
      6 #include "base/callback.h"
      7 #include "base/compiler_specific.h"
      8 #include "base/memory/weak_ptr.h"
      9 #include "base/message_loop/message_loop.h"
     10 #include "base/test/test_timeouts.h"
     11 #include "sync/engine/backoff_delay_provider.h"
     12 #include "sync/engine/sync_scheduler_impl.h"
     13 #include "sync/engine/syncer.h"
     14 #include "sync/internal_api/public/base/cancelation_signal.h"
     15 #include "sync/internal_api/public/base/model_type_test_util.h"
     16 #include "sync/notifier/invalidation_util.h"
     17 #include "sync/notifier/object_id_invalidation_map.h"
     18 #include "sync/sessions/test_util.h"
     19 #include "sync/test/callback_counter.h"
     20 #include "sync/test/engine/fake_model_worker.h"
     21 #include "sync/test/engine/mock_connection_manager.h"
     22 #include "sync/test/engine/test_directory_setter_upper.h"
     23 #include "sync/util/extensions_activity.h"
     24 #include "testing/gmock/include/gmock/gmock.h"
     25 #include "testing/gtest/include/gtest/gtest.h"
     26 
     27 using base::TimeDelta;
     28 using base::TimeTicks;
     29 using testing::_;
     30 using testing::AtLeast;
     31 using testing::DoAll;
     32 using testing::Invoke;
     33 using testing::Mock;
     34 using testing::Return;
     35 using testing::WithArg;
     36 using testing::WithArgs;
     37 using testing::WithoutArgs;
     38 
     39 namespace syncer {
     40 using sessions::SyncSession;
     41 using sessions::SyncSessionContext;
     42 using sync_pb::GetUpdatesCallerInfo;
     43 
     44 class MockSyncer : public Syncer {
     45  public:
     46   MockSyncer();
     47   MOCK_METHOD3(NormalSyncShare, bool(ModelTypeSet,
     48                                      const sessions::NudgeTracker&,
     49                                      sessions::SyncSession*));
     50   MOCK_METHOD3(ConfigureSyncShare,
     51                bool(ModelTypeSet,
     52                     sync_pb::GetUpdatesCallerInfo::GetUpdatesSource,
     53                     SyncSession*));
     54   MOCK_METHOD2(PollSyncShare, bool(ModelTypeSet, sessions::SyncSession*));
     55 };
     56 
     57 MockSyncer::MockSyncer()
     58   : Syncer(NULL) {}
     59 
     60 typedef std::vector<TimeTicks> SyncShareTimes;
     61 
     62 void QuitLoopNow() {
     63   // We use QuitNow() instead of Quit() as the latter may get stalled
     64   // indefinitely in the presence of repeated timers with low delays
     65   // and a slow test (e.g., ThrottlingDoesThrottle [which has a poll
     66   // delay of 5ms] run under TSAN on the trybots).
     67   base::MessageLoop::current()->QuitNow();
     68 }
     69 
     70 void RunLoop() {
     71   base::MessageLoop::current()->Run();
     72 }
     73 
     74 void PumpLoop() {
     75   // Do it this way instead of RunAllPending to pump loop exactly once
     76   // (necessary in the presence of timers; see comment in
     77   // QuitLoopNow).
     78   base::MessageLoop::current()->PostTask(FROM_HERE, base::Bind(&QuitLoopNow));
     79   RunLoop();
     80 }
     81 
     82 void PumpLoopFor(base::TimeDelta time) {
     83   // Allow the loop to run for the specified amount of time.
     84   base::MessageLoop::current()->PostDelayedTask(
     85       FROM_HERE, base::Bind(&QuitLoopNow), time);
     86   RunLoop();
     87 }
     88 
     89 ModelSafeRoutingInfo TypesToRoutingInfo(ModelTypeSet types) {
     90   ModelSafeRoutingInfo routes;
     91   for (ModelTypeSet::Iterator iter = types.First(); iter.Good(); iter.Inc()) {
     92     routes[iter.Get()] = GROUP_PASSIVE;
     93   }
     94   return routes;
     95 }
     96 
     97 // Convenient to use in tests wishing to analyze SyncShare calls over time.
     98 static const size_t kMinNumSamples = 5;
     99 class SyncSchedulerTest : public testing::Test {
    100  public:
    101   SyncSchedulerTest() : syncer_(NULL), delay_(NULL), weak_ptr_factory_(this) {}
    102 
    103   class MockDelayProvider : public BackoffDelayProvider {
    104    public:
    105     MockDelayProvider() : BackoffDelayProvider(
    106         TimeDelta::FromSeconds(kInitialBackoffRetrySeconds),
    107         TimeDelta::FromSeconds(kInitialBackoffImmediateRetrySeconds)) {
    108     }
    109 
    110     MOCK_METHOD1(GetDelay, TimeDelta(const TimeDelta&));
    111   };
    112 
    113   virtual void SetUp() {
    114     dir_maker_.SetUp();
    115     syncer_ = new testing::StrictMock<MockSyncer>();
    116     delay_ = NULL;
    117     extensions_activity_ = new ExtensionsActivity();
    118 
    119     routing_info_[BOOKMARKS] = GROUP_UI;
    120     routing_info_[AUTOFILL] = GROUP_DB;
    121     routing_info_[THEMES] = GROUP_UI;
    122     routing_info_[NIGORI] = GROUP_PASSIVE;
    123 
    124     workers_.clear();
    125     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_UI)));
    126     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_DB)));
    127     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_PASSIVE)));
    128 
    129     connection_.reset(new MockConnectionManager(directory(),
    130                                                 &cancelation_signal_));
    131     connection_->SetServerReachable();
    132 
    133     model_type_registry_.reset(new ModelTypeRegistry(workers_, directory()));
    134 
    135     context_.reset(new SyncSessionContext(
    136             connection_.get(), directory(),
    137             extensions_activity_.get(),
    138             std::vector<SyncEngineEventListener*>(), NULL,
    139             model_type_registry_.get(),
    140             true,  // enable keystore encryption
    141             false,  // force enable pre-commit GU avoidance
    142             "fake_invalidator_client_id"));
    143     context_->SetRoutingInfo(routing_info_);
    144     context_->set_notifications_enabled(true);
    145     context_->set_account_name("Test");
    146     scheduler_.reset(
    147         new SyncSchedulerImpl("TestSyncScheduler",
    148             BackoffDelayProvider::FromDefaults(),
    149             context(),
    150             syncer_));
    151   }
    152 
    153   SyncSchedulerImpl* scheduler() { return scheduler_.get(); }
    154   const ModelSafeRoutingInfo& routing_info() { return routing_info_; }
    155   MockSyncer* syncer() { return syncer_; }
    156   MockDelayProvider* delay() { return delay_; }
    157   MockConnectionManager* connection() { return connection_.get(); }
    158   TimeDelta zero() { return TimeDelta::FromSeconds(0); }
    159   TimeDelta timeout() {
    160     return TestTimeouts::action_timeout();
    161   }
    162 
    163   virtual void TearDown() {
    164     PumpLoop();
    165     scheduler_.reset();
    166     PumpLoop();
    167     dir_maker_.TearDown();
    168   }
    169 
    170   void AnalyzePollRun(const SyncShareTimes& times, size_t min_num_samples,
    171       const TimeTicks& optimal_start, const TimeDelta& poll_interval) {
    172     EXPECT_GE(times.size(), min_num_samples);
    173     for (size_t i = 0; i < times.size(); i++) {
    174       SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
    175       TimeTicks optimal_next_sync = optimal_start + poll_interval * i;
    176       EXPECT_GE(times[i], optimal_next_sync);
    177     }
    178   }
    179 
    180   void DoQuitLoopNow() {
    181     QuitLoopNow();
    182   }
    183 
    184   void StartSyncScheduler(SyncScheduler::Mode mode) {
    185     scheduler()->Start(mode);
    186   }
    187 
    188   // This stops the scheduler synchronously.
    189   void StopSyncScheduler() {
    190     base::MessageLoop::current()->PostTask(
    191         FROM_HERE,
    192         base::Bind(&SyncSchedulerTest::DoQuitLoopNow,
    193                    weak_ptr_factory_.GetWeakPtr()));
    194     RunLoop();
    195   }
    196 
    197   bool RunAndGetBackoff() {
    198     ModelTypeSet nudge_types(BOOKMARKS);
    199     StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    200 
    201     scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
    202     RunLoop();
    203 
    204     return scheduler()->IsBackingOff();
    205   }
    206 
    207   void UseMockDelayProvider() {
    208     delay_ = new MockDelayProvider();
    209     scheduler_->delay_provider_.reset(delay_);
    210   }
    211 
    212   SyncSessionContext* context() { return context_.get(); }
    213 
    214   ModelTypeSet GetThrottledTypes() {
    215     return scheduler_->nudge_tracker_.GetThrottledTypes();
    216   }
    217 
    218   base::TimeDelta GetRetryTimerDelay() {
    219     EXPECT_TRUE(scheduler_->retry_timer_.IsRunning());
    220     return scheduler_->retry_timer_.GetCurrentDelay();
    221   }
    222 
    223  private:
    224   syncable::Directory* directory() {
    225     return dir_maker_.directory();
    226   }
    227 
    228   base::MessageLoop loop_;
    229   TestDirectorySetterUpper dir_maker_;
    230   CancelationSignal cancelation_signal_;
    231   scoped_ptr<MockConnectionManager> connection_;
    232   scoped_ptr<ModelTypeRegistry> model_type_registry_;
    233   scoped_ptr<SyncSessionContext> context_;
    234   scoped_ptr<SyncSchedulerImpl> scheduler_;
    235   MockSyncer* syncer_;
    236   MockDelayProvider* delay_;
    237   std::vector<scoped_refptr<ModelSafeWorker> > workers_;
    238   scoped_refptr<ExtensionsActivity> extensions_activity_;
    239   ModelSafeRoutingInfo routing_info_;
    240   base::WeakPtrFactory<SyncSchedulerTest> weak_ptr_factory_;
    241 };
    242 
    243 void RecordSyncShareImpl(SyncShareTimes* times) {
    244   times->push_back(TimeTicks::Now());
    245 }
    246 
    247 ACTION_P(RecordSyncShare, times) {
    248   RecordSyncShareImpl(times);
    249   if (base::MessageLoop::current()->is_running())
    250     QuitLoopNow();
    251   return true;
    252 }
    253 
    254 ACTION_P2(RecordSyncShareMultiple, times, quit_after) {
    255   RecordSyncShareImpl(times);
    256   EXPECT_LE(times->size(), quit_after);
    257   if (times->size() >= quit_after &&
    258       base::MessageLoop::current()->is_running()) {
    259     QuitLoopNow();
    260   }
    261   return true;
    262 }
    263 
    264 ACTION_P(StopScheduler, scheduler) {
    265   scheduler->Stop();
    266 }
    267 
    268 ACTION(AddFailureAndQuitLoopNow) {
    269   ADD_FAILURE();
    270   QuitLoopNow();
    271   return true;
    272 }
    273 
    274 ACTION(QuitLoopNowAction) {
    275   QuitLoopNow();
    276   return true;
    277 }
    278 
    279 // Test nudge scheduling.
    280 TEST_F(SyncSchedulerTest, Nudge) {
    281   SyncShareTimes times;
    282   ModelTypeSet model_types(BOOKMARKS);
    283 
    284   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    285       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    286                       RecordSyncShare(&times)))
    287       .RetiresOnSaturation();
    288 
    289   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    290 
    291   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
    292   RunLoop();
    293 
    294   Mock::VerifyAndClearExpectations(syncer());
    295 
    296   // Make sure a second, later, nudge is unaffected by first (no coalescing).
    297   SyncShareTimes times2;
    298   model_types.Remove(BOOKMARKS);
    299   model_types.Put(AUTOFILL);
    300   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    301       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    302                       RecordSyncShare(&times2)));
    303   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
    304   RunLoop();
    305 }
    306 
    307 // Make sure a regular config command is scheduled fine in the absence of any
    308 // errors.
    309 TEST_F(SyncSchedulerTest, Config) {
    310   SyncShareTimes times;
    311   const ModelTypeSet model_types(BOOKMARKS);
    312 
    313   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    314       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
    315                       RecordSyncShare(&times)));
    316 
    317   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    318 
    319   CallbackCounter ready_counter;
    320   CallbackCounter retry_counter;
    321   ConfigurationParams params(
    322       GetUpdatesCallerInfo::RECONFIGURATION,
    323       model_types,
    324       TypesToRoutingInfo(model_types),
    325       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
    326       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
    327   scheduler()->ScheduleConfiguration(params);
    328   PumpLoop();
    329   ASSERT_EQ(1, ready_counter.times_called());
    330   ASSERT_EQ(0, retry_counter.times_called());
    331 }
    332 
    333 // Simulate a failure and make sure the config request is retried.
    334 TEST_F(SyncSchedulerTest, ConfigWithBackingOff) {
    335   UseMockDelayProvider();
    336   EXPECT_CALL(*delay(), GetDelay(_))
    337       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
    338   SyncShareTimes times;
    339   const ModelTypeSet model_types(BOOKMARKS);
    340 
    341   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    342 
    343   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    344       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
    345                       RecordSyncShare(&times)))
    346       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
    347                       RecordSyncShare(&times)));
    348 
    349   CallbackCounter ready_counter;
    350   CallbackCounter retry_counter;
    351   ConfigurationParams params(
    352       GetUpdatesCallerInfo::RECONFIGURATION,
    353       model_types,
    354       TypesToRoutingInfo(model_types),
    355       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
    356       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
    357   scheduler()->ScheduleConfiguration(params);
    358   RunLoop();
    359   ASSERT_EQ(0, ready_counter.times_called());
    360   ASSERT_EQ(1, retry_counter.times_called());
    361 
    362   // RunLoop() will trigger TryCanaryJob which will retry configuration.
    363   // Since retry_task was already called it shouldn't be called again.
    364   RunLoop();
    365   ASSERT_EQ(0, ready_counter.times_called());
    366   ASSERT_EQ(1, retry_counter.times_called());
    367 
    368   Mock::VerifyAndClearExpectations(syncer());
    369 
    370   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    371       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
    372                       RecordSyncShare(&times)));
    373   RunLoop();
    374 
    375   ASSERT_EQ(1, ready_counter.times_called());
    376 }
    377 
    378 // Simuilate SyncSchedulerImpl::Stop being called in the middle of Configure.
    379 // This can happen if server returns NOT_MY_BIRTHDAY.
    380 TEST_F(SyncSchedulerTest, ConfigWithStop) {
    381   UseMockDelayProvider();
    382   EXPECT_CALL(*delay(), GetDelay(_))
    383       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
    384   SyncShareTimes times;
    385   const ModelTypeSet model_types(BOOKMARKS);
    386 
    387   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    388 
    389   // Make ConfigureSyncShare call scheduler->Stop(). It is not supposed to call
    390   // retry_task or dereference configuration params.
    391   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    392       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
    393                       StopScheduler(scheduler()),
    394                       RecordSyncShare(&times)));
    395 
    396   CallbackCounter ready_counter;
    397   CallbackCounter retry_counter;
    398   ConfigurationParams params(
    399       GetUpdatesCallerInfo::RECONFIGURATION,
    400       model_types,
    401       TypesToRoutingInfo(model_types),
    402       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
    403       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
    404   scheduler()->ScheduleConfiguration(params);
    405   PumpLoop();
    406   ASSERT_EQ(0, ready_counter.times_called());
    407   ASSERT_EQ(0, retry_counter.times_called());
    408 }
    409 
    410 // Issue a nudge when the config has failed. Make sure both the config and
    411 // nudge are executed.
    412 TEST_F(SyncSchedulerTest, NudgeWithConfigWithBackingOff) {
    413   const ModelTypeSet model_types(BOOKMARKS);
    414   UseMockDelayProvider();
    415   EXPECT_CALL(*delay(), GetDelay(_))
    416       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(50)));
    417   SyncShareTimes times;
    418 
    419   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    420 
    421   // Request a configure and make sure it fails.
    422   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    423       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
    424                       RecordSyncShare(&times)));
    425   CallbackCounter ready_counter;
    426   CallbackCounter retry_counter;
    427   ConfigurationParams params(
    428       GetUpdatesCallerInfo::RECONFIGURATION,
    429       model_types,
    430       TypesToRoutingInfo(model_types),
    431       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
    432       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
    433   scheduler()->ScheduleConfiguration(params);
    434   RunLoop();
    435   ASSERT_EQ(0, ready_counter.times_called());
    436   ASSERT_EQ(1, retry_counter.times_called());
    437   Mock::VerifyAndClearExpectations(syncer());
    438 
    439   // Ask for a nudge while dealing with repeated configure failure.
    440   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    441       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
    442                       RecordSyncShare(&times)));
    443   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
    444   RunLoop();
    445   // Note that we're not RunLoop()ing for the NUDGE we just scheduled, but
    446   // for the first retry attempt from the config job (after
    447   // waiting ~+/- 50ms).
    448   Mock::VerifyAndClearExpectations(syncer());
    449   ASSERT_EQ(0, ready_counter.times_called());
    450 
    451   // Let the next configure retry succeed.
    452   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    453       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
    454                       RecordSyncShare(&times)));
    455   RunLoop();
    456 
    457   // Now change the mode so nudge can execute.
    458   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    459       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    460                       RecordSyncShare(&times)));
    461   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    462   PumpLoop();
    463 }
    464 
    465 // Test that nudges are coalesced.
    466 TEST_F(SyncSchedulerTest, NudgeCoalescing) {
    467   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    468 
    469   SyncShareTimes times;
    470   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    471       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    472                       RecordSyncShare(&times)));
    473   const ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3(THEMES);
    474   TimeDelta delay = zero();
    475   TimeTicks optimal_time = TimeTicks::Now() + delay;
    476   scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
    477   scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
    478   RunLoop();
    479 
    480   ASSERT_EQ(1U, times.size());
    481   EXPECT_GE(times[0], optimal_time);
    482 
    483   Mock::VerifyAndClearExpectations(syncer());
    484 
    485   SyncShareTimes times2;
    486   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    487       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    488                       RecordSyncShare(&times2)));
    489   scheduler()->ScheduleLocalNudge(zero(), types3, FROM_HERE);
    490   RunLoop();
    491 }
    492 
    493 // Test that nudges are coalesced.
    494 TEST_F(SyncSchedulerTest, NudgeCoalescingWithDifferentTimings) {
    495   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    496 
    497   SyncShareTimes times;
    498   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    499       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    500                       RecordSyncShare(&times)));
    501   ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3;
    502 
    503   // Create a huge time delay.
    504   TimeDelta delay = TimeDelta::FromDays(1);
    505 
    506   scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
    507   scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
    508 
    509   TimeTicks min_time = TimeTicks::Now();
    510   TimeTicks max_time = TimeTicks::Now() + delay;
    511 
    512   RunLoop();
    513   Mock::VerifyAndClearExpectations(syncer());
    514 
    515   // Make sure the sync happened at the right time.
    516   ASSERT_EQ(1U, times.size());
    517   EXPECT_GE(times[0], min_time);
    518   EXPECT_LE(times[0], max_time);
    519 }
    520 
    521 // Test nudge scheduling.
    522 TEST_F(SyncSchedulerTest, NudgeWithStates) {
    523   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    524 
    525   SyncShareTimes times1;
    526   ObjectIdInvalidationMap invalidations1 =
    527       BuildInvalidationMap(BOOKMARKS, 10, "test");
    528   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    529       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    530                       RecordSyncShare(&times1)))
    531       .RetiresOnSaturation();
    532   scheduler()->ScheduleInvalidationNudge(zero(), invalidations1, FROM_HERE);
    533   RunLoop();
    534 
    535   Mock::VerifyAndClearExpectations(syncer());
    536 
    537   // Make sure a second, later, nudge is unaffected by first (no coalescing).
    538   SyncShareTimes times2;
    539   ObjectIdInvalidationMap invalidations2 =
    540       BuildInvalidationMap(AUTOFILL, 10, "test2");
    541   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    542       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    543                       RecordSyncShare(&times2)));
    544   scheduler()->ScheduleInvalidationNudge(zero(), invalidations2, FROM_HERE);
    545   RunLoop();
    546 }
    547 
    548 // Test that polling works as expected.
    549 TEST_F(SyncSchedulerTest, Polling) {
    550   SyncShareTimes times;
    551   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
    552   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
    553       .WillRepeatedly(
    554           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
    555                 RecordSyncShareMultiple(&times, kMinNumSamples)));
    556 
    557   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
    558 
    559   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
    560   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    561 
    562   // Run again to wait for polling.
    563   RunLoop();
    564 
    565   StopSyncScheduler();
    566   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
    567 }
    568 
    569 // Test that the short poll interval is used.
    570 TEST_F(SyncSchedulerTest, PollNotificationsDisabled) {
    571   SyncShareTimes times;
    572   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
    573   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
    574       .WillRepeatedly(
    575           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
    576                 RecordSyncShareMultiple(&times, kMinNumSamples)));
    577 
    578   scheduler()->OnReceivedShortPollIntervalUpdate(poll_interval);
    579   scheduler()->SetNotificationsEnabled(false);
    580 
    581   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
    582   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    583 
    584   // Run again to wait for polling.
    585   RunLoop();
    586 
    587   StopSyncScheduler();
    588   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
    589 }
    590 
    591 // Test that polling intervals are updated when needed.
    592 TEST_F(SyncSchedulerTest, PollIntervalUpdate) {
    593   SyncShareTimes times;
    594   TimeDelta poll1(TimeDelta::FromMilliseconds(120));
    595   TimeDelta poll2(TimeDelta::FromMilliseconds(30));
    596   scheduler()->OnReceivedLongPollIntervalUpdate(poll1);
    597   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
    598       .WillOnce(DoAll(
    599           WithArgs<0,1>(
    600               sessions::test_util::SimulatePollIntervalUpdate(poll2)),
    601           Return(true)))
    602       .WillRepeatedly(
    603           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
    604                 WithArg<1>(
    605                     RecordSyncShareMultiple(&times, kMinNumSamples))));
    606 
    607   TimeTicks optimal_start = TimeTicks::Now() + poll1 + poll2;
    608   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    609 
    610   // Run again to wait for polling.
    611   RunLoop();
    612 
    613   StopSyncScheduler();
    614   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll2);
    615 }
    616 
    617 // Test that the sessions commit delay is updated when needed.
    618 TEST_F(SyncSchedulerTest, SessionsCommitDelay) {
    619   SyncShareTimes times;
    620   TimeDelta delay1(TimeDelta::FromMilliseconds(120));
    621   TimeDelta delay2(TimeDelta::FromMilliseconds(30));
    622   scheduler()->OnReceivedSessionsCommitDelay(delay1);
    623 
    624   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    625       .WillOnce(
    626           DoAll(
    627               WithArgs<0,1,2>(
    628                   sessions::test_util::SimulateSessionsCommitDelayUpdate(
    629                       delay2)),
    630               Invoke(sessions::test_util::SimulateNormalSuccess),
    631               QuitLoopNowAction()));
    632 
    633   EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
    634   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    635 
    636   EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
    637   const ModelTypeSet model_types(BOOKMARKS);
    638   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
    639   RunLoop();
    640 
    641   EXPECT_EQ(delay2, scheduler()->GetSessionsCommitDelay());
    642   StopSyncScheduler();
    643 }
    644 
    645 // Test that no syncing occurs when throttled.
    646 TEST_F(SyncSchedulerTest, ThrottlingDoesThrottle) {
    647   const ModelTypeSet types(BOOKMARKS);
    648   TimeDelta poll(TimeDelta::FromMilliseconds(5));
    649   TimeDelta throttle(TimeDelta::FromMinutes(10));
    650   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    651 
    652   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    653       .WillOnce(DoAll(
    654           WithArg<2>(sessions::test_util::SimulateThrottled(throttle)),
    655           Return(true)))
    656       .WillRepeatedly(AddFailureAndQuitLoopNow());
    657 
    658   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    659 
    660   scheduler()->ScheduleLocalNudge(
    661       TimeDelta::FromMicroseconds(1), types, FROM_HERE);
    662   PumpLoop();
    663 
    664   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    665 
    666   CallbackCounter ready_counter;
    667   CallbackCounter retry_counter;
    668   ConfigurationParams params(
    669       GetUpdatesCallerInfo::RECONFIGURATION,
    670       types,
    671       TypesToRoutingInfo(types),
    672       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
    673       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
    674   scheduler()->ScheduleConfiguration(params);
    675   PumpLoop();
    676   ASSERT_EQ(0, ready_counter.times_called());
    677   ASSERT_EQ(1, retry_counter.times_called());
    678 
    679 }
    680 
    681 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromPoll) {
    682   SyncShareTimes times;
    683   TimeDelta poll(TimeDelta::FromMilliseconds(15));
    684   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
    685   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    686 
    687   ::testing::InSequence seq;
    688   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
    689       .WillOnce(DoAll(
    690           WithArg<1>(sessions::test_util::SimulateThrottled(throttle1)),
    691           Return(true)))
    692       .RetiresOnSaturation();
    693   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
    694       .WillRepeatedly(
    695           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
    696                 RecordSyncShareMultiple(&times, kMinNumSamples)));
    697 
    698   TimeTicks optimal_start = TimeTicks::Now() + poll + throttle1;
    699   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    700 
    701   // Run again to wait for polling.
    702   RunLoop();
    703 
    704   StopSyncScheduler();
    705   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll);
    706 }
    707 
    708 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromNudge) {
    709   SyncShareTimes times;
    710   TimeDelta poll(TimeDelta::FromDays(1));
    711   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
    712   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    713 
    714   ::testing::InSequence seq;
    715   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    716       .WillOnce(DoAll(
    717           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
    718           Return(true)))
    719       .RetiresOnSaturation();
    720   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    721       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    722                       QuitLoopNowAction()));
    723 
    724   const ModelTypeSet types(BOOKMARKS);
    725   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    726   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
    727 
    728   PumpLoop(); // To get PerformDelayedNudge called.
    729   PumpLoop(); // To get TrySyncSessionJob called
    730   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
    731   RunLoop();
    732   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
    733 
    734   StopSyncScheduler();
    735 }
    736 
    737 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromConfigure) {
    738   SyncShareTimes times;
    739   TimeDelta poll(TimeDelta::FromDays(1));
    740   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
    741   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    742 
    743   ::testing::InSequence seq;
    744   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    745       .WillOnce(DoAll(
    746           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
    747           Return(true)))
    748       .RetiresOnSaturation();
    749   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    750       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
    751                       QuitLoopNowAction()));
    752 
    753   const ModelTypeSet types(BOOKMARKS);
    754   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    755 
    756   CallbackCounter ready_counter;
    757   CallbackCounter retry_counter;
    758   ConfigurationParams params(
    759       GetUpdatesCallerInfo::RECONFIGURATION,
    760       types,
    761       TypesToRoutingInfo(types),
    762       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
    763       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
    764   scheduler()->ScheduleConfiguration(params);
    765   PumpLoop();
    766   EXPECT_EQ(0, ready_counter.times_called());
    767   EXPECT_EQ(1, retry_counter.times_called());
    768   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
    769 
    770   RunLoop();
    771   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
    772 
    773   StopSyncScheduler();
    774 }
    775 
    776 TEST_F(SyncSchedulerTest, TypeThrottlingBlocksNudge) {
    777   UseMockDelayProvider();
    778   EXPECT_CALL(*delay(), GetDelay(_))
    779       .WillRepeatedly(Return(zero()));
    780 
    781   TimeDelta poll(TimeDelta::FromDays(1));
    782   TimeDelta throttle1(TimeDelta::FromSeconds(60));
    783   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    784 
    785   const ModelTypeSet types(BOOKMARKS);
    786 
    787   ::testing::InSequence seq;
    788   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    789       .WillOnce(DoAll(
    790           WithArg<2>(
    791               sessions::test_util::SimulateTypesThrottled(types, throttle1)),
    792           Return(true)))
    793       .RetiresOnSaturation();
    794 
    795   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    796   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
    797   PumpLoop(); // To get PerformDelayedNudge called.
    798   PumpLoop(); // To get TrySyncSessionJob called
    799   EXPECT_TRUE(GetThrottledTypes().HasAll(types));
    800 
    801   // This won't cause a sync cycle because the types are throttled.
    802   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
    803   PumpLoop();
    804 
    805   StopSyncScheduler();
    806 }
    807 
    808 TEST_F(SyncSchedulerTest, TypeThrottlingDoesBlockOtherSources) {
    809   UseMockDelayProvider();
    810   EXPECT_CALL(*delay(), GetDelay(_))
    811       .WillRepeatedly(Return(zero()));
    812 
    813   SyncShareTimes times;
    814   TimeDelta poll(TimeDelta::FromDays(1));
    815   TimeDelta throttle1(TimeDelta::FromSeconds(60));
    816   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    817 
    818   const ModelTypeSet throttled_types(BOOKMARKS);
    819   const ModelTypeSet unthrottled_types(PREFERENCES);
    820 
    821   ::testing::InSequence seq;
    822   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    823       .WillOnce(DoAll(
    824           WithArg<2>(
    825               sessions::test_util::SimulateTypesThrottled(
    826                   throttled_types, throttle1)),
    827           Return(true)))
    828       .RetiresOnSaturation();
    829 
    830   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    831   scheduler()->ScheduleLocalNudge(zero(), throttled_types, FROM_HERE);
    832   PumpLoop(); // To get PerformDelayedNudge called.
    833   PumpLoop(); // To get TrySyncSessionJob called
    834   EXPECT_TRUE(GetThrottledTypes().HasAll(throttled_types));
    835 
    836   // Ignore invalidations for throttled types.
    837   ObjectIdInvalidationMap invalidations =
    838       BuildInvalidationMap(BOOKMARKS, 10, "test");
    839   scheduler()->ScheduleInvalidationNudge(zero(), invalidations, FROM_HERE);
    840   PumpLoop();
    841 
    842   // Ignore refresh requests for throttled types.
    843   scheduler()->ScheduleLocalRefreshRequest(zero(), throttled_types, FROM_HERE);
    844   PumpLoop();
    845 
    846   Mock::VerifyAndClearExpectations(syncer());
    847 
    848   // Local nudges for non-throttled types will trigger a sync.
    849   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    850       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    851                             RecordSyncShare(&times)));
    852   scheduler()->ScheduleLocalNudge(zero(), unthrottled_types, FROM_HERE);
    853   RunLoop();
    854   Mock::VerifyAndClearExpectations(syncer());
    855 
    856   StopSyncScheduler();
    857 }
    858 
    859 // Test nudges / polls don't run in config mode and config tasks do.
    860 TEST_F(SyncSchedulerTest, ConfigurationMode) {
    861   TimeDelta poll(TimeDelta::FromMilliseconds(15));
    862   SyncShareTimes times;
    863   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
    864 
    865   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    866 
    867   const ModelTypeSet nudge_types(AUTOFILL);
    868   scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
    869   scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
    870 
    871   const ModelTypeSet config_types(BOOKMARKS);
    872 
    873   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    874       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
    875                       RecordSyncShare(&times)))
    876       .RetiresOnSaturation();
    877   CallbackCounter ready_counter;
    878   CallbackCounter retry_counter;
    879   ConfigurationParams params(
    880       GetUpdatesCallerInfo::RECONFIGURATION,
    881       config_types,
    882       TypesToRoutingInfo(config_types),
    883       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
    884       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
    885   scheduler()->ScheduleConfiguration(params);
    886   RunLoop();
    887   ASSERT_EQ(1, ready_counter.times_called());
    888   ASSERT_EQ(0, retry_counter.times_called());
    889 
    890   Mock::VerifyAndClearExpectations(syncer());
    891 
    892   // Switch to NORMAL_MODE to ensure NUDGES were properly saved and run.
    893   scheduler()->OnReceivedLongPollIntervalUpdate(TimeDelta::FromDays(1));
    894   SyncShareTimes times2;
    895   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    896       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    897                       RecordSyncShare(&times2)));
    898 
    899   // TODO(tim): Figure out how to remove this dangerous need to reset
    900   // routing info between mode switches.
    901   context()->SetRoutingInfo(routing_info());
    902   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
    903 
    904   RunLoop();
    905   Mock::VerifyAndClearExpectations(syncer());
    906 }
    907 
    908 class BackoffTriggersSyncSchedulerTest : public SyncSchedulerTest {
    909   virtual void SetUp() {
    910     SyncSchedulerTest::SetUp();
    911     UseMockDelayProvider();
    912     EXPECT_CALL(*delay(), GetDelay(_))
    913         .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
    914   }
    915 
    916   virtual void TearDown() {
    917     StopSyncScheduler();
    918     SyncSchedulerTest::TearDown();
    919   }
    920 };
    921 
    922 // Have the sycner fail during commit.  Expect that the scheduler enters
    923 // backoff.
    924 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnce) {
    925   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    926       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
    927                       QuitLoopNowAction()));
    928   EXPECT_TRUE(RunAndGetBackoff());
    929 }
    930 
    931 // Have the syncer fail during download updates and succeed on the first
    932 // retry.  Expect that this clears the backoff state.
    933 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadOnceThenSucceed) {
    934   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    935       .WillOnce(DoAll(
    936           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
    937           Return(true)))
    938       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    939                       QuitLoopNowAction()));
    940   EXPECT_FALSE(RunAndGetBackoff());
    941 }
    942 
    943 // Have the syncer fail during commit and succeed on the first retry.  Expect
    944 // that this clears the backoff state.
    945 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnceThenSucceed) {
    946   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    947       .WillOnce(DoAll(
    948           Invoke(sessions::test_util::SimulateCommitFailed),
    949           Return(true)))
    950       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
    951                       QuitLoopNowAction()));
    952   EXPECT_FALSE(RunAndGetBackoff());
    953 }
    954 
    955 // Have the syncer fail to download updates and fail again on the retry.
    956 // Expect this will leave the scheduler in backoff.
    957 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadTwice) {
    958   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
    959       .WillOnce(DoAll(
    960           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
    961           Return(true)))
    962       .WillRepeatedly(DoAll(
    963               Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
    964               QuitLoopNowAction()));
    965   EXPECT_TRUE(RunAndGetBackoff());
    966 }
    967 
    968 // Have the syncer fail to get the encryption key yet succeed in downloading
    969 // updates. Expect this will leave the scheduler in backoff.
    970 TEST_F(BackoffTriggersSyncSchedulerTest, FailGetEncryptionKey) {
    971   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
    972       .WillOnce(DoAll(
    973           Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
    974           Return(true)))
    975       .WillRepeatedly(DoAll(
    976               Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
    977               QuitLoopNowAction()));
    978   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
    979 
    980   ModelTypeSet types(BOOKMARKS);
    981   CallbackCounter ready_counter;
    982   CallbackCounter retry_counter;
    983   ConfigurationParams params(
    984       GetUpdatesCallerInfo::RECONFIGURATION,
    985       types,
    986       TypesToRoutingInfo(types),
    987       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
    988       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
    989   scheduler()->ScheduleConfiguration(params);
    990   RunLoop();
    991 
    992   EXPECT_TRUE(scheduler()->IsBackingOff());
    993 }
    994 
    995 // Test that no polls or extraneous nudges occur when in backoff.
    996 TEST_F(SyncSchedulerTest, BackoffDropsJobs) {
    997   SyncShareTimes times;
    998   TimeDelta poll(TimeDelta::FromMilliseconds(5));
    999   const ModelTypeSet types(BOOKMARKS);
   1000   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
   1001   UseMockDelayProvider();
   1002 
   1003   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1004       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
   1005                       RecordSyncShareMultiple(&times, 1U)));
   1006   EXPECT_CALL(*delay(), GetDelay(_)).
   1007       WillRepeatedly(Return(TimeDelta::FromDays(1)));
   1008 
   1009   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1010 
   1011   // This nudge should fail and put us into backoff.  Thanks to our mock
   1012   // GetDelay() setup above, this will be a long backoff.
   1013   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
   1014   RunLoop();
   1015 
   1016   // From this point forward, no SyncShare functions should be invoked.
   1017   Mock::VerifyAndClearExpectations(syncer());
   1018 
   1019   // Wait a while (10x poll interval) so a few poll jobs will be attempted.
   1020   PumpLoopFor(poll * 10);
   1021 
   1022   // Try (and fail) to schedule a nudge.
   1023   scheduler()->ScheduleLocalNudge(
   1024       base::TimeDelta::FromMilliseconds(1),
   1025       types,
   1026       FROM_HERE);
   1027 
   1028   Mock::VerifyAndClearExpectations(syncer());
   1029   Mock::VerifyAndClearExpectations(delay());
   1030 
   1031   EXPECT_CALL(*delay(), GetDelay(_)).Times(0);
   1032 
   1033   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
   1034 
   1035   CallbackCounter ready_counter;
   1036   CallbackCounter retry_counter;
   1037   ConfigurationParams params(
   1038       GetUpdatesCallerInfo::RECONFIGURATION,
   1039       types,
   1040       TypesToRoutingInfo(types),
   1041       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
   1042       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
   1043   scheduler()->ScheduleConfiguration(params);
   1044   PumpLoop();
   1045   ASSERT_EQ(0, ready_counter.times_called());
   1046   ASSERT_EQ(1, retry_counter.times_called());
   1047 
   1048 }
   1049 
   1050 // Test that backoff is shaping traffic properly with consecutive errors.
   1051 TEST_F(SyncSchedulerTest, BackoffElevation) {
   1052   SyncShareTimes times;
   1053   UseMockDelayProvider();
   1054 
   1055   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_)).Times(kMinNumSamples)
   1056       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
   1057           RecordSyncShareMultiple(&times, kMinNumSamples)));
   1058 
   1059   const TimeDelta first = TimeDelta::FromSeconds(kInitialBackoffRetrySeconds);
   1060   const TimeDelta second = TimeDelta::FromMilliseconds(2);
   1061   const TimeDelta third = TimeDelta::FromMilliseconds(3);
   1062   const TimeDelta fourth = TimeDelta::FromMilliseconds(4);
   1063   const TimeDelta fifth = TimeDelta::FromMilliseconds(5);
   1064   const TimeDelta sixth = TimeDelta::FromDays(1);
   1065 
   1066   EXPECT_CALL(*delay(), GetDelay(first)).WillOnce(Return(second))
   1067           .RetiresOnSaturation();
   1068   EXPECT_CALL(*delay(), GetDelay(second)).WillOnce(Return(third))
   1069           .RetiresOnSaturation();
   1070   EXPECT_CALL(*delay(), GetDelay(third)).WillOnce(Return(fourth))
   1071           .RetiresOnSaturation();
   1072   EXPECT_CALL(*delay(), GetDelay(fourth)).WillOnce(Return(fifth))
   1073           .RetiresOnSaturation();
   1074   EXPECT_CALL(*delay(), GetDelay(fifth)).WillOnce(Return(sixth));
   1075 
   1076   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1077 
   1078   // Run again with a nudge.
   1079   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
   1080   RunLoop();
   1081 
   1082   ASSERT_EQ(kMinNumSamples, times.size());
   1083   EXPECT_GE(times[1] - times[0], second);
   1084   EXPECT_GE(times[2] - times[1], third);
   1085   EXPECT_GE(times[3] - times[2], fourth);
   1086   EXPECT_GE(times[4] - times[3], fifth);
   1087 }
   1088 
   1089 // Test that things go back to normal once a retry makes forward progress.
   1090 TEST_F(SyncSchedulerTest, BackoffRelief) {
   1091   SyncShareTimes times;
   1092   const TimeDelta poll(TimeDelta::FromMilliseconds(10));
   1093   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
   1094   UseMockDelayProvider();
   1095 
   1096   const TimeDelta backoff = TimeDelta::FromMilliseconds(5);
   1097   EXPECT_CALL(*delay(), GetDelay(_)).WillOnce(Return(backoff));
   1098 
   1099   // Optimal start for the post-backoff poll party.
   1100   TimeTicks optimal_start = TimeTicks::Now();
   1101   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1102 
   1103   // Kick off the test with a failed nudge.
   1104   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1105       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
   1106                       RecordSyncShare(&times)));
   1107   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
   1108   RunLoop();
   1109   Mock::VerifyAndClearExpectations(syncer());
   1110   TimeTicks optimal_job_time = optimal_start;
   1111   ASSERT_EQ(1U, times.size());
   1112   EXPECT_GE(times[0], optimal_job_time);
   1113 
   1114   // The retry succeeds.
   1115   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1116       .WillOnce(DoAll(
   1117               Invoke(sessions::test_util::SimulateNormalSuccess),
   1118               RecordSyncShare(&times)));
   1119   RunLoop();
   1120   Mock::VerifyAndClearExpectations(syncer());
   1121   optimal_job_time = optimal_job_time + backoff;
   1122   ASSERT_EQ(2U, times.size());
   1123   EXPECT_GE(times[1], optimal_job_time);
   1124 
   1125   // Now let the Poll timer do its thing.
   1126   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
   1127       .WillRepeatedly(DoAll(
   1128               Invoke(sessions::test_util::SimulatePollSuccess),
   1129               RecordSyncShareMultiple(&times, kMinNumSamples)));
   1130   RunLoop();
   1131   Mock::VerifyAndClearExpectations(syncer());
   1132   ASSERT_EQ(kMinNumSamples, times.size());
   1133   for (size_t i = 2; i < times.size(); i++) {
   1134     optimal_job_time = optimal_job_time + poll;
   1135     SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
   1136     EXPECT_GE(times[i], optimal_job_time);
   1137   }
   1138 
   1139   StopSyncScheduler();
   1140 }
   1141 
   1142 // Test that poll failures are ignored.  They should have no effect on
   1143 // subsequent poll attempts, nor should they trigger a backoff/retry.
   1144 TEST_F(SyncSchedulerTest, TransientPollFailure) {
   1145   SyncShareTimes times;
   1146   const TimeDelta poll_interval(TimeDelta::FromMilliseconds(1));
   1147   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
   1148   UseMockDelayProvider(); // Will cause test failure if backoff is initiated.
   1149 
   1150   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
   1151       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollFailed),
   1152                       RecordSyncShare(&times)))
   1153       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
   1154                       RecordSyncShare(&times)));
   1155 
   1156   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1157 
   1158   // Run the unsucessful poll. The failed poll should not trigger backoff.
   1159   RunLoop();
   1160   EXPECT_FALSE(scheduler()->IsBackingOff());
   1161 
   1162   // Run the successful poll.
   1163   RunLoop();
   1164   EXPECT_FALSE(scheduler()->IsBackingOff());
   1165 }
   1166 
   1167 // Test that starting the syncer thread without a valid connection doesn't
   1168 // break things when a connection is detected.
   1169 TEST_F(SyncSchedulerTest, StartWhenNotConnected) {
   1170   connection()->SetServerNotReachable();
   1171   connection()->UpdateConnectionStatus();
   1172   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1173     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
   1174                     Return(true)))
   1175     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
   1176                     Return(true)));
   1177   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1178 
   1179   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
   1180   // Should save the nudge for until after the server is reachable.
   1181   base::MessageLoop::current()->RunUntilIdle();
   1182 
   1183   scheduler()->OnConnectionStatusChange();
   1184   connection()->SetServerReachable();
   1185   connection()->UpdateConnectionStatus();
   1186   base::MessageLoop::current()->RunUntilIdle();
   1187 }
   1188 
   1189 TEST_F(SyncSchedulerTest, ServerConnectionChangeDuringBackoff) {
   1190   UseMockDelayProvider();
   1191   EXPECT_CALL(*delay(), GetDelay(_))
   1192       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
   1193 
   1194   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1195   connection()->SetServerNotReachable();
   1196   connection()->UpdateConnectionStatus();
   1197 
   1198   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1199     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
   1200                     Return(true)))
   1201     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
   1202                     Return(true)));
   1203 
   1204   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
   1205   PumpLoop(); // To get PerformDelayedNudge called.
   1206   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
   1207   ASSERT_TRUE(scheduler()->IsBackingOff());
   1208 
   1209   // Before we run the scheduled canary, trigger a server connection change.
   1210   scheduler()->OnConnectionStatusChange();
   1211   connection()->SetServerReachable();
   1212   connection()->UpdateConnectionStatus();
   1213   base::MessageLoop::current()->RunUntilIdle();
   1214 }
   1215 
   1216 // This was supposed to test the scenario where we receive a nudge while a
   1217 // connection change canary is scheduled, but has not run yet.  Since we've made
   1218 // the connection change canary synchronous, this is no longer possible.
   1219 TEST_F(SyncSchedulerTest, ConnectionChangeCanaryPreemptedByNudge) {
   1220   UseMockDelayProvider();
   1221   EXPECT_CALL(*delay(), GetDelay(_))
   1222       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
   1223 
   1224   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1225   connection()->SetServerNotReachable();
   1226   connection()->UpdateConnectionStatus();
   1227 
   1228   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1229     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
   1230                     Return(true)))
   1231     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
   1232                     Return(true)))
   1233     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
   1234                     QuitLoopNowAction()));
   1235 
   1236   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
   1237 
   1238   PumpLoop(); // To get PerformDelayedNudge called.
   1239   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
   1240   ASSERT_TRUE(scheduler()->IsBackingOff());
   1241 
   1242   // Before we run the scheduled canary, trigger a server connection change.
   1243   scheduler()->OnConnectionStatusChange();
   1244   PumpLoop();
   1245   connection()->SetServerReachable();
   1246   connection()->UpdateConnectionStatus();
   1247   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
   1248   base::MessageLoop::current()->RunUntilIdle();
   1249 }
   1250 
   1251 // Tests that we don't crash trying to run two canaries at once if we receive
   1252 // extra connection status change notifications.  See crbug.com/190085.
   1253 TEST_F(SyncSchedulerTest, DoubleCanaryInConfigure) {
   1254   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
   1255       .WillRepeatedly(DoAll(
   1256               Invoke(sessions::test_util::SimulateConfigureConnectionFailure),
   1257               Return(true)));
   1258   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
   1259   connection()->SetServerNotReachable();
   1260   connection()->UpdateConnectionStatus();
   1261 
   1262   ModelTypeSet model_types(BOOKMARKS);
   1263   CallbackCounter ready_counter;
   1264   CallbackCounter retry_counter;
   1265   ConfigurationParams params(
   1266       GetUpdatesCallerInfo::RECONFIGURATION,
   1267       model_types,
   1268       TypesToRoutingInfo(model_types),
   1269       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
   1270       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
   1271   scheduler()->ScheduleConfiguration(params);
   1272 
   1273   scheduler()->OnConnectionStatusChange();
   1274   scheduler()->OnConnectionStatusChange();
   1275 
   1276   PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
   1277 }
   1278 
   1279 TEST_F(SyncSchedulerTest, PollFromCanaryAfterAuthError) {
   1280   SyncShareTimes times;
   1281   TimeDelta poll(TimeDelta::FromMilliseconds(15));
   1282   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
   1283 
   1284   ::testing::InSequence seq;
   1285   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
   1286       .WillRepeatedly(
   1287           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
   1288                 RecordSyncShareMultiple(&times, kMinNumSamples)));
   1289 
   1290   connection()->SetServerStatus(HttpResponse::SYNC_AUTH_ERROR);
   1291   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1292 
   1293   // Run to wait for polling.
   1294   RunLoop();
   1295 
   1296   // Normally OnCredentialsUpdated calls TryCanaryJob that doesn't run Poll,
   1297   // but after poll finished with auth error from poll timer it should retry
   1298   // poll once more
   1299   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
   1300       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
   1301                       RecordSyncShare(&times)));
   1302   scheduler()->OnCredentialsUpdated();
   1303   connection()->SetServerStatus(HttpResponse::SERVER_CONNECTION_OK);
   1304   RunLoop();
   1305   StopSyncScheduler();
   1306 }
   1307 
   1308 TEST_F(SyncSchedulerTest, SuccessfulRetry) {
   1309   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1310 
   1311   SyncShareTimes times;
   1312   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(1);
   1313   scheduler()->OnReceivedGuRetryDelay(delay);
   1314   EXPECT_EQ(delay, GetRetryTimerDelay());
   1315 
   1316   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1317       .WillOnce(
   1318           DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
   1319                 RecordSyncShare(&times)));
   1320 
   1321   // Run to wait for retrying.
   1322   RunLoop();
   1323 
   1324   StopSyncScheduler();
   1325 }
   1326 
   1327 TEST_F(SyncSchedulerTest, FailedRetry) {
   1328   UseMockDelayProvider();
   1329   EXPECT_CALL(*delay(), GetDelay(_))
   1330       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
   1331 
   1332   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1333 
   1334   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(1);
   1335   scheduler()->OnReceivedGuRetryDelay(delay);
   1336 
   1337   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1338       .WillOnce(
   1339           DoAll(Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
   1340                 QuitLoopNowAction()));
   1341 
   1342   // Run to wait for retrying.
   1343   RunLoop();
   1344 
   1345   EXPECT_TRUE(scheduler()->IsBackingOff());
   1346   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1347       .WillOnce(
   1348           DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
   1349                 QuitLoopNowAction()));
   1350 
   1351   // Run to wait for second retrying.
   1352   RunLoop();
   1353 
   1354   StopSyncScheduler();
   1355 }
   1356 
   1357 ACTION_P2(VerifyRetryTimerDelay, scheduler_test, expected_delay) {
   1358   EXPECT_EQ(expected_delay, scheduler_test->GetRetryTimerDelay());
   1359 }
   1360 
   1361 TEST_F(SyncSchedulerTest, ReceiveNewRetryDelay) {
   1362   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
   1363 
   1364   SyncShareTimes times;
   1365   base::TimeDelta delay1 = base::TimeDelta::FromMilliseconds(100);
   1366   base::TimeDelta delay2 = base::TimeDelta::FromMilliseconds(200);
   1367 
   1368   scheduler()->ScheduleLocalRefreshRequest(zero(), ModelTypeSet(BOOKMARKS),
   1369                                            FROM_HERE);
   1370   scheduler()->OnReceivedGuRetryDelay(delay1);
   1371   EXPECT_EQ(delay1, GetRetryTimerDelay());
   1372 
   1373   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1374       .WillOnce(DoAll(
   1375           WithoutArgs(VerifyRetryTimerDelay(this, delay1)),
   1376           WithArg<2>(sessions::test_util::SimulateGuRetryDelayCommand(delay2)),
   1377           RecordSyncShare(&times)));
   1378 
   1379   // Run nudge GU.
   1380   RunLoop();
   1381   EXPECT_EQ(delay2, GetRetryTimerDelay());
   1382 
   1383   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
   1384       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
   1385                       RecordSyncShare(&times)));
   1386 
   1387   // Run to wait for retrying.
   1388   RunLoop();
   1389 
   1390   StopSyncScheduler();
   1391 }
   1392 
   1393 }  // namespace syncer
   1394