Home | History | Annotate | Download | only in variations 
      1 // Copyright 2014 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "components/variations/variations_seed_simulator.h"
      6 
      7 #include <map>
      8 
      9 #include "base/strings/stringprintf.h"
     10 #include "components/variations/processed_study.h"
     11 #include "components/variations/proto/study.pb.h"
     12 #include "components/variations/variations_associated_data.h"
     13 #include "testing/gtest/include/gtest/gtest.h"
     14 
     15 namespace variations {
     16 
     17 namespace {
     18 
     19 // An implementation of EntropyProvider that always returns a specific entropy
     20 // value, regardless of field trial.
     21 class TestEntropyProvider : public base::FieldTrial::EntropyProvider {
     22  public:
     23   explicit TestEntropyProvider(double entropy_value)
     24       : entropy_value_(entropy_value) {}
     25   virtual ~TestEntropyProvider() {}
     26 
     27   // base::FieldTrial::EntropyProvider implementation:
     28   virtual double GetEntropyForTrial(const std::string& trial_name,
     29                                     uint32 randomization_seed) const OVERRIDE {
     30     return entropy_value_;
     31   }
     32 
     33  private:
     34   const double entropy_value_;
     35 
     36   DISALLOW_COPY_AND_ASSIGN(TestEntropyProvider);
     37 };
     38 
     39 // Creates and activates a single-group field trial with name |trial_name| and
     40 // group |group_name| and variations |params| (if not NULL).
     41 void CreateTrial(const std::string& trial_name,
     42                  const std::string& group_name,
     43                  const std::map<std::string, std::string>* params) {
     44   base::FieldTrialList::CreateFieldTrial(trial_name, group_name);
     45   if (params != NULL)
     46     AssociateVariationParams(trial_name, group_name, *params);
     47   base::FieldTrialList::FindFullName(trial_name);
     48 }
     49 
     50 // Creates a study with the given |study_name| and |consistency|.
     51 Study CreateStudy(const std::string& study_name,
     52                   Study_Consistency consistency) {
     53   Study study;
     54   study.set_name(study_name);
     55   study.set_consistency(consistency);
     56   return study;
     57 }
     58 
     59 // Adds an experiment to |study| with the specified |experiment_name| and
     60 // |probability| values and sets it as the study's default experiment.
     61 Study_Experiment* AddExperiment(const std::string& experiment_name,
     62                                 int probability,
     63                                 Study* study) {
     64   Study_Experiment* experiment = study->add_experiment();
     65   experiment->set_name(experiment_name);
     66   experiment->set_probability_weight(probability);
     67   study->set_default_experiment_name(experiment_name);
     68   return experiment;
     69 }
     70 
     71 // Add an experiment param with |param_name| and |param_value| to |experiment|.
     72 Study_Experiment_Param* AddExperimentParam(const std::string& param_name,
     73                                            const std::string& param_value,
     74                                            Study_Experiment* experiment) {
     75   Study_Experiment_Param* param = experiment->add_param();
     76   param->set_name(param_name);
     77   param->set_value(param_value);
     78   return param;
     79 }
     80 
     81 }  // namespace
     82 
     83 class VariationsSeedSimulatorTest : public ::testing::Test {
     84  public:
     85   VariationsSeedSimulatorTest() : field_trial_list_(NULL) {
     86   }
     87 
     88   virtual ~VariationsSeedSimulatorTest() {
     89     // Ensure that the maps are cleared between tests, since they are stored as
     90     // process singletons.
     91     testing::ClearAllVariationIDs();
     92     testing::ClearAllVariationParams();
     93   }
     94 
     95   // Uses a VariationsSeedSimulator to simulate the differences between
     96   // |studies| and the current field trial state.
     97   VariationsSeedSimulator::Result SimulateDifferences(
     98       const std::vector<ProcessedStudy>& studies) {
     99     TestEntropyProvider provider(0.5);
    100     VariationsSeedSimulator seed_simulator(provider);
    101     return seed_simulator.ComputeDifferences(studies);
    102   }
    103 
    104   // Simulates the differences between |study| and the current field trial
    105   // state, returning a string like "1 2 3", where 1 is the number of regular
    106   // group changes, 2 is the number of "kill best effort" group changes and 3
    107   // is the number of "kill critical" group changes.
    108   std::string SimulateStudyDifferences(const Study* study) {
    109     std::vector<ProcessedStudy> studies;
    110     if (!ProcessedStudy::ValidateAndAppendStudy(study, false, &studies))
    111       return "invalid study";
    112     return ConvertSimulationResultToString(SimulateDifferences(studies));
    113   }
    114 
    115   // Simulates the differences between expired |study| and the current field
    116   // trial state, returning a string like "1 2 3", where 1 is the number of
    117   // regular group changes, 2 is the number of "kill best effort" group changes
    118   // and 3 is the number of "kill critical" group changes.
    119   std::string SimulateStudyDifferencesExpired(const Study* study) {
    120     std::vector<ProcessedStudy> studies;
    121     if (!ProcessedStudy::ValidateAndAppendStudy(study, true, &studies))
    122       return "invalid study";
    123     if (!studies[0].is_expired())
    124       return "not expired";
    125     return ConvertSimulationResultToString(SimulateDifferences(studies));
    126   }
    127 
    128   // Formats |result| as a string with format "1 2 3", where 1 is the number of
    129   // regular group changes, 2 is the number of "kill best effort" group changes
    130   // and 3 is the number of "kill critical" group changes.
    131   std::string ConvertSimulationResultToString(
    132       const VariationsSeedSimulator::Result& result) {
    133     return base::StringPrintf("%d %d %d",
    134                               result.normal_group_change_count,
    135                               result.kill_best_effort_group_change_count,
    136                               result.kill_critical_group_change_count);
    137   }
    138 
    139  private:
    140   base::FieldTrialList field_trial_list_;
    141 
    142   DISALLOW_COPY_AND_ASSIGN(VariationsSeedSimulatorTest);
    143 };
    144 
    145 TEST_F(VariationsSeedSimulatorTest, PermanentNoChanges) {
    146   CreateTrial("A", "B", NULL);
    147 
    148   std::vector<ProcessedStudy> processed_studies;
    149   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
    150   Study_Experiment* experiment = AddExperiment("B", 100, &study);
    151 
    152   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    153 
    154   experiment->set_type(Study_Experiment_Type_NORMAL);
    155   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    156   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    157   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    158   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    159   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    160   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    161   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    162 }
    163 
    164 TEST_F(VariationsSeedSimulatorTest, PermanentGroupChange) {
    165   CreateTrial("A", "B", NULL);
    166 
    167   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
    168   Study_Experiment* experiment = AddExperiment("C", 100, &study);
    169 
    170   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    171 
    172   // Changing "C" group type should not affect the type of change. (Since the
    173   // type is evaluated for the "old" group.)
    174   experiment->set_type(Study_Experiment_Type_NORMAL);
    175   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    176   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    177   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    178   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    179   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    180   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    181   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    182 }
    183 
    184 TEST_F(VariationsSeedSimulatorTest, PermanentExpired) {
    185   CreateTrial("A", "B", NULL);
    186 
    187   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
    188   Study_Experiment* experiment = AddExperiment("B", 1, &study);
    189   AddExperiment("C", 0, &study);
    190 
    191   // There should be a difference because the study is expired, which should
    192   // result in the default group "C" being chosen.
    193   EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
    194 
    195   experiment->set_type(Study_Experiment_Type_NORMAL);
    196   EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
    197   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    198   EXPECT_EQ("0 0 0", SimulateStudyDifferencesExpired(&study));
    199   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    200   EXPECT_EQ("0 1 0", SimulateStudyDifferencesExpired(&study));
    201   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    202   EXPECT_EQ("0 0 1", SimulateStudyDifferencesExpired(&study));
    203 }
    204 
    205 TEST_F(VariationsSeedSimulatorTest, SessionRandomized) {
    206   CreateTrial("A", "B", NULL);
    207 
    208   Study study = CreateStudy("A", Study_Consistency_SESSION);
    209   Study_Experiment* experiment = AddExperiment("B", 1, &study);
    210   AddExperiment("C", 1, &study);
    211   AddExperiment("D", 1, &study);
    212 
    213   // There should be no differences, since a session randomized study can result
    214   // in any of the groups being chosen on startup.
    215   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    216 
    217   experiment->set_type(Study_Experiment_Type_NORMAL);
    218   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    219   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    220   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    221   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    222   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    223   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    224   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    225 }
    226 
    227 TEST_F(VariationsSeedSimulatorTest, SessionRandomizedGroupRemoved) {
    228   CreateTrial("A", "B", NULL);
    229 
    230   Study study = CreateStudy("A", Study_Consistency_SESSION);
    231   AddExperiment("C", 1, &study);
    232   AddExperiment("D", 1, &study);
    233 
    234   // There should be a difference since there is no group "B" in the new config.
    235   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    236 }
    237 
    238 TEST_F(VariationsSeedSimulatorTest, SessionRandomizedGroupProbabilityZero) {
    239   CreateTrial("A", "B", NULL);
    240 
    241   Study study = CreateStudy("A", Study_Consistency_SESSION);
    242   Study_Experiment* experiment = AddExperiment("B", 0, &study);
    243   AddExperiment("C", 1, &study);
    244   AddExperiment("D", 1, &study);
    245 
    246   // There should be a difference since group "B" has probability 0.
    247   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    248 
    249   experiment->set_type(Study_Experiment_Type_NORMAL);
    250   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    251   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    252   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    253   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    254   EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
    255   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    256   EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
    257 }
    258 
    259 TEST_F(VariationsSeedSimulatorTest, SessionRandomizedExpired) {
    260   CreateTrial("A", "B", NULL);
    261 
    262   Study study = CreateStudy("A", Study_Consistency_SESSION);
    263   Study_Experiment* experiment = AddExperiment("B", 1, &study);
    264   AddExperiment("C", 1, &study);
    265   AddExperiment("D", 1, &study);
    266 
    267   // There should be a difference because the study is expired, which should
    268   // result in the default group "D" being chosen.
    269   EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
    270 
    271   experiment->set_type(Study_Experiment_Type_NORMAL);
    272   EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
    273   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    274   EXPECT_EQ("0 0 0", SimulateStudyDifferencesExpired(&study));
    275   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    276   EXPECT_EQ("0 1 0", SimulateStudyDifferencesExpired(&study));
    277   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    278   EXPECT_EQ("0 0 1", SimulateStudyDifferencesExpired(&study));
    279 }
    280 
    281 TEST_F(VariationsSeedSimulatorTest, ParamsUnchanged) {
    282   std::map<std::string, std::string> params;
    283   params["p1"] = "x";
    284   params["p2"] = "y";
    285   params["p3"] = "z";
    286   CreateTrial("A", "B", &params);
    287 
    288   std::vector<ProcessedStudy> processed_studies;
    289   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
    290   Study_Experiment* experiment = AddExperiment("B", 100, &study);
    291   AddExperimentParam("p2", "y", experiment);
    292   AddExperimentParam("p1", "x", experiment);
    293   AddExperimentParam("p3", "z", experiment);
    294 
    295   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    296 
    297   experiment->set_type(Study_Experiment_Type_NORMAL);
    298   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    299   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    300   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    301   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    302   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    303   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    304   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    305 }
    306 
    307 TEST_F(VariationsSeedSimulatorTest, ParamsChanged) {
    308   std::map<std::string, std::string> params;
    309   params["p1"] = "x";
    310   params["p2"] = "y";
    311   params["p3"] = "z";
    312   CreateTrial("A", "B", &params);
    313 
    314   std::vector<ProcessedStudy> processed_studies;
    315   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
    316   Study_Experiment* experiment = AddExperiment("B", 100, &study);
    317   AddExperimentParam("p2", "test", experiment);
    318   AddExperimentParam("p1", "x", experiment);
    319   AddExperimentParam("p3", "z", experiment);
    320 
    321   // The param lists differ.
    322   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    323 
    324   experiment->set_type(Study_Experiment_Type_NORMAL);
    325   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    326   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    327   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    328   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    329   EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
    330   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    331   EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
    332 }
    333 
    334 TEST_F(VariationsSeedSimulatorTest, ParamsRemoved) {
    335   std::map<std::string, std::string> params;
    336   params["p1"] = "x";
    337   params["p2"] = "y";
    338   params["p3"] = "z";
    339   CreateTrial("A", "B", &params);
    340 
    341   std::vector<ProcessedStudy> processed_studies;
    342   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
    343   Study_Experiment* experiment = AddExperiment("B", 100, &study);
    344 
    345   // The current group has params, but the new config doesn't have any.
    346   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    347 
    348   experiment->set_type(Study_Experiment_Type_NORMAL);
    349   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    350   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    351   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    352   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    353   EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
    354   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    355   EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
    356 }
    357 
    358 TEST_F(VariationsSeedSimulatorTest, ParamsAdded) {
    359   CreateTrial("A", "B", NULL);
    360 
    361   std::vector<ProcessedStudy> processed_studies;
    362   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
    363   Study_Experiment* experiment = AddExperiment("B", 100, &study);
    364   AddExperimentParam("p2", "y", experiment);
    365   AddExperimentParam("p1", "x", experiment);
    366   AddExperimentParam("p3", "z", experiment);
    367 
    368   // The current group has no params, but the config has added some.
    369   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    370 
    371   experiment->set_type(Study_Experiment_Type_NORMAL);
    372   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    373   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    374   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    375   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    376   EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
    377   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    378   EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
    379 }
    380 
    381 }  // namespace variations
    382