Home | History | Annotate | Download | only in variations 
      1 // Copyright 2014 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "components/variations/variations_seed_simulator.h"
      6 
      7 #include <map>
      8 
      9 #include "base/strings/stringprintf.h"
     10 #include "components/variations/processed_study.h"
     11 #include "components/variations/proto/study.pb.h"
     12 #include "components/variations/variations_associated_data.h"
     13 #include "testing/gtest/include/gtest/gtest.h"
     14 
     15 namespace chrome_variations {
     16 
     17 namespace {
     18 
     19 // An implementation of EntropyProvider that always returns a specific entropy
     20 // value, regardless of field trial.
     21 class TestEntropyProvider : public base::FieldTrial::EntropyProvider {
     22  public:
     23   explicit TestEntropyProvider(double entropy_value)
     24       : entropy_value_(entropy_value) {}
     25   virtual ~TestEntropyProvider() {}
     26 
     27   // base::FieldTrial::EntropyProvider implementation:
     28   virtual double GetEntropyForTrial(const std::string& trial_name,
     29                                     uint32 randomization_seed) const OVERRIDE {
     30     return entropy_value_;
     31   }
     32 
     33  private:
     34   const double entropy_value_;
     35 
     36   DISALLOW_COPY_AND_ASSIGN(TestEntropyProvider);
     37 };
     38 
     39 // Creates and activates a single-group field trial with name |trial_name| and
     40 // group |group_name| and variations |params| (if not NULL).
     41 void CreateTrial(const std::string& trial_name,
     42                  const std::string& group_name,
     43                  const std::map<std::string, std::string>* params) {
     44   base::FieldTrialList::CreateFieldTrial(trial_name, group_name);
     45   if (params != NULL)
     46     AssociateVariationParams(trial_name, group_name, *params);
     47   base::FieldTrialList::FindFullName(trial_name);
     48 }
     49 
     50 // Creates a study with the given |study_name| and |consistency|.
     51 Study CreateStudy(const std::string& study_name,
     52                   Study_Consistency consistency) {
     53   Study study;
     54   study.set_name(study_name);
     55   study.set_consistency(consistency);
     56   return study;
     57 }
     58 
     59 // Adds an experiment to |study| with the specified |experiment_name| and
     60 // |probability| values and sets it as the study's default experiment.
     61 Study_Experiment* AddExperiment(const std::string& experiment_name,
     62                                 int probability,
     63                                 Study* study) {
     64   Study_Experiment* experiment = study->add_experiment();
     65   experiment->set_name(experiment_name);
     66   experiment->set_probability_weight(probability);
     67   study->set_default_experiment_name(experiment_name);
     68   return experiment;
     69 }
     70 
     71 // Add an experiment param with |param_name| and |param_value| to |experiment|.
     72 Study_Experiment_Param* AddExperimentParam(const std::string& param_name,
     73                                            const std::string& param_value,
     74                                            Study_Experiment* experiment) {
     75   Study_Experiment_Param* param = experiment->add_param();
     76   param->set_name(param_name);
     77   param->set_value(param_value);
     78   return param;
     79 }
     80 
     81 }  // namespace
     82 
     83 class VariationsSeedSimulatorTest : public ::testing::Test {
     84  public:
     85   VariationsSeedSimulatorTest() : field_trial_list_(NULL) {
     86   }
     87 
     88   virtual ~VariationsSeedSimulatorTest() {
     89     // Ensure that the maps are cleared between tests, since they are stored as
     90     // process singletons.
     91     testing::ClearAllVariationIDs();
     92     testing::ClearAllVariationParams();
     93   }
     94 
     95   // Uses a VariationsSeedSimulator to simulate the differences between
     96   // |studies| and the current field trial state.
     97   VariationsSeedSimulator::Result SimulateDifferences(
     98       const std::vector<ProcessedStudy>& studies) {
     99     TestEntropyProvider provider(0.5);
    100     VariationsSeedSimulator seed_simulator(provider);
    101     return seed_simulator.ComputeDifferences(studies);
    102   }
    103 
    104   // Simulates the differences between |study| and the current field trial
    105   // state, returning a string like "1 2 3", where 1 is the number of regular
    106   // group changes, 2 is the number of "kill best effort" group changes and 3
    107   // is the number of "kill critical" group changes.
    108   std::string SimulateStudyDifferences(const Study* study) {
    109     std::vector<ProcessedStudy> studies;
    110     if (!ProcessedStudy::ValidateAndAppendStudy(study, false, &studies))
    111       return "invalid study";
    112     return ConvertSimulationResultToString(SimulateDifferences(studies));
    113 
    114   }
    115 
    116   // Simulates the differences between expired |study| and the current field
    117   // trial state, returning a string like "1 2 3", where 1 is the number of
    118   // regular group changes, 2 is the number of "kill best effort" group changes
    119   // and 3 is the number of "kill critical" group changes.
    120   std::string SimulateStudyDifferencesExpired(const Study* study) {
    121     std::vector<ProcessedStudy> studies;
    122     if (!ProcessedStudy::ValidateAndAppendStudy(study, true, &studies))
    123       return "invalid study";
    124     if (!studies[0].is_expired())
    125       return "not expired";
    126     return ConvertSimulationResultToString(SimulateDifferences(studies));
    127   }
    128 
    129   // Formats |result| as a string with format "1 2 3", where 1 is the number of
    130   // regular group changes, 2 is the number of "kill best effort" group changes
    131   // and 3 is the number of "kill critical" group changes.
    132   std::string ConvertSimulationResultToString(
    133       const VariationsSeedSimulator::Result& result) {
    134     return base::StringPrintf("%d %d %d",
    135                               result.normal_group_change_count,
    136                               result.kill_best_effort_group_change_count,
    137                               result.kill_critical_group_change_count);
    138   }
    139 
    140  private:
    141   base::FieldTrialList field_trial_list_;
    142 
    143   DISALLOW_COPY_AND_ASSIGN(VariationsSeedSimulatorTest);
    144 };
    145 
    146 TEST_F(VariationsSeedSimulatorTest, PermanentNoChanges) {
    147   CreateTrial("A", "B", NULL);
    148 
    149   std::vector<ProcessedStudy> processed_studies;
    150   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
    151   Study_Experiment* experiment = AddExperiment("B", 100, &study);
    152 
    153   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    154 
    155   experiment->set_type(Study_Experiment_Type_NORMAL);
    156   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    157   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    158   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    159   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    160   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    161   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    162   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    163 }
    164 
    165 TEST_F(VariationsSeedSimulatorTest, PermanentGroupChange) {
    166   CreateTrial("A", "B", NULL);
    167 
    168   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
    169   Study_Experiment* experiment = AddExperiment("C", 100, &study);
    170 
    171   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    172 
    173   // Changing "C" group type should not affect the type of change. (Since the
    174   // type is evaluated for the "old" group.)
    175   experiment->set_type(Study_Experiment_Type_NORMAL);
    176   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    177   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    178   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    179   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    180   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    181   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    182   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    183 }
    184 
    185 TEST_F(VariationsSeedSimulatorTest, PermanentExpired) {
    186   CreateTrial("A", "B", NULL);
    187 
    188   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
    189   Study_Experiment* experiment = AddExperiment("B", 1, &study);
    190   AddExperiment("C", 0, &study);
    191 
    192   // There should be a difference because the study is expired, which should
    193   // result in the default group "C" being chosen.
    194   EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
    195 
    196   experiment->set_type(Study_Experiment_Type_NORMAL);
    197   EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
    198   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    199   EXPECT_EQ("0 0 0", SimulateStudyDifferencesExpired(&study));
    200   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    201   EXPECT_EQ("0 1 0", SimulateStudyDifferencesExpired(&study));
    202   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    203   EXPECT_EQ("0 0 1", SimulateStudyDifferencesExpired(&study));
    204 }
    205 
    206 TEST_F(VariationsSeedSimulatorTest, SessionRandomized) {
    207   CreateTrial("A", "B", NULL);
    208 
    209   Study study = CreateStudy("A", Study_Consistency_SESSION);
    210   Study_Experiment* experiment = AddExperiment("B", 1, &study);
    211   AddExperiment("C", 1, &study);
    212   AddExperiment("D", 1, &study);
    213 
    214   // There should be no differences, since a session randomized study can result
    215   // in any of the groups being chosen on startup.
    216   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    217 
    218   experiment->set_type(Study_Experiment_Type_NORMAL);
    219   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    220   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    221   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    222   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    223   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    224   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    225   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    226 }
    227 
    228 TEST_F(VariationsSeedSimulatorTest, SessionRandomizedGroupRemoved) {
    229   CreateTrial("A", "B", NULL);
    230 
    231   Study study = CreateStudy("A", Study_Consistency_SESSION);
    232   AddExperiment("C", 1, &study);
    233   AddExperiment("D", 1, &study);
    234 
    235   // There should be a difference since there is no group "B" in the new config.
    236   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    237 }
    238 
    239 TEST_F(VariationsSeedSimulatorTest, SessionRandomizedGroupProbabilityZero) {
    240   CreateTrial("A", "B", NULL);
    241 
    242   Study study = CreateStudy("A", Study_Consistency_SESSION);
    243   Study_Experiment* experiment = AddExperiment("B", 0, &study);
    244   AddExperiment("C", 1, &study);
    245   AddExperiment("D", 1, &study);
    246 
    247   // There should be a difference since group "B" has probability 0.
    248   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    249 
    250   experiment->set_type(Study_Experiment_Type_NORMAL);
    251   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    252   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    253   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    254   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    255   EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
    256   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    257   EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
    258 }
    259 
    260 TEST_F(VariationsSeedSimulatorTest, SessionRandomizedExpired) {
    261   CreateTrial("A", "B", NULL);
    262 
    263   Study study = CreateStudy("A", Study_Consistency_SESSION);
    264   Study_Experiment* experiment = AddExperiment("B", 1, &study);
    265   AddExperiment("C", 1, &study);
    266   AddExperiment("D", 1, &study);
    267 
    268   // There should be a difference because the study is expired, which should
    269   // result in the default group "D" being chosen.
    270   EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
    271 
    272   experiment->set_type(Study_Experiment_Type_NORMAL);
    273   EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
    274   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    275   EXPECT_EQ("0 0 0", SimulateStudyDifferencesExpired(&study));
    276   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    277   EXPECT_EQ("0 1 0", SimulateStudyDifferencesExpired(&study));
    278   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    279   EXPECT_EQ("0 0 1", SimulateStudyDifferencesExpired(&study));
    280 }
    281 
    282 TEST_F(VariationsSeedSimulatorTest, ParamsUnchanged) {
    283   std::map<std::string, std::string> params;
    284   params["p1"] = "x";
    285   params["p2"] = "y";
    286   params["p3"] = "z";
    287   CreateTrial("A", "B", &params);
    288 
    289   std::vector<ProcessedStudy> processed_studies;
    290   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
    291   Study_Experiment* experiment = AddExperiment("B", 100, &study);
    292   AddExperimentParam("p2", "y", experiment);
    293   AddExperimentParam("p1", "x", experiment);
    294   AddExperimentParam("p3", "z", experiment);
    295 
    296   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    297 
    298   experiment->set_type(Study_Experiment_Type_NORMAL);
    299   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    300   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    301   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    302   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    303   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    304   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    305   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    306 }
    307 
    308 TEST_F(VariationsSeedSimulatorTest, ParamsChanged) {
    309   std::map<std::string, std::string> params;
    310   params["p1"] = "x";
    311   params["p2"] = "y";
    312   params["p3"] = "z";
    313   CreateTrial("A", "B", &params);
    314 
    315   std::vector<ProcessedStudy> processed_studies;
    316   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
    317   Study_Experiment* experiment = AddExperiment("B", 100, &study);
    318   AddExperimentParam("p2", "test", experiment);
    319   AddExperimentParam("p1", "x", experiment);
    320   AddExperimentParam("p3", "z", experiment);
    321 
    322   // The param lists differ.
    323   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    324 
    325   experiment->set_type(Study_Experiment_Type_NORMAL);
    326   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    327   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    328   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    329   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    330   EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
    331   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    332   EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
    333 }
    334 
    335 TEST_F(VariationsSeedSimulatorTest, ParamsRemoved) {
    336   std::map<std::string, std::string> params;
    337   params["p1"] = "x";
    338   params["p2"] = "y";
    339   params["p3"] = "z";
    340   CreateTrial("A", "B", &params);
    341 
    342   std::vector<ProcessedStudy> processed_studies;
    343   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
    344   Study_Experiment* experiment = AddExperiment("B", 100, &study);
    345 
    346   // The current group has params, but the new config doesn't have any.
    347   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    348 
    349   experiment->set_type(Study_Experiment_Type_NORMAL);
    350   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    351   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    352   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    353   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    354   EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
    355   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    356   EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
    357 }
    358 
    359 TEST_F(VariationsSeedSimulatorTest, ParamsAdded) {
    360   CreateTrial("A", "B", NULL);
    361 
    362   std::vector<ProcessedStudy> processed_studies;
    363   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
    364   Study_Experiment* experiment = AddExperiment("B", 100, &study);
    365   AddExperimentParam("p2", "y", experiment);
    366   AddExperimentParam("p1", "x", experiment);
    367   AddExperimentParam("p3", "z", experiment);
    368 
    369   // The current group has no params, but the config has added some.
    370   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    371 
    372   experiment->set_type(Study_Experiment_Type_NORMAL);
    373   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
    374   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
    375   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
    376   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
    377   EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
    378   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
    379   EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
    380 }
    381 
    382 }  // namespace chrome_variations
    383